In tilestats, truncate and merge layers and attributes with long names

mbtiles.cpp

@@ -142,7 +142,12 @@ bool type_and_string::operator!=(const type_and_string &o) const {
 	return false;
 }
 
-std::string tilestats(std::map<std::string, layermap_entry> const &layermap) {
+std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
+	// Consolidate layers/attributes whose names are truncated
+	std::vector<std::map<std::string, layermap_entry>> lmv;
+	lmv.push_back(layermap1);
+	std::map<std::string, layermap_entry> layermap = merge_layermaps(lmv, true);
+
 	std::string out = "{\n";
 
 	out.append("\t\"layerCount\": ");
@@ -535,28 +540,42 @@ void mbtiles_close(sqlite3 *outdb, char **argv) {
 }
 
 std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry>> const &maps) {
+	return merge_layermaps(maps, false);
+}
+
+std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry>> const &maps, bool trunc) {
 	std::map<std::string, layermap_entry> out;
 
 	for (size_t i = 0; i < maps.size(); i++) {
 		for (auto map = maps[i].begin(); map != maps[i].end(); ++map) {
-			if (out.count(map->first) == 0) {
+			std::string layername = map->first;
-				out.insert(std::pair<std::string, layermap_entry>(map->first, layermap_entry(out.size())));
+			if (trunc) {
-				auto out_entry = out.find(map->first);
+				layername = truncate16(layername, 256);
+			}
+
+			if (out.count(layername) == 0) {
+				out.insert(std::pair<std::string, layermap_entry>(layername, layermap_entry(out.size())));
+				auto out_entry = out.find(layername);
 				out_entry->second.minzoom = map->second.minzoom;
 				out_entry->second.maxzoom = map->second.maxzoom;
 			}
 
-			auto out_entry = out.find(map->first);
+			auto out_entry = out.find(layername);
 			if (out_entry == out.end()) {
 				fprintf(stderr, "Internal error merging layers\n");
 				exit(EXIT_FAILURE);
 			}
 
 			for (auto fk = map->second.file_keys.begin(); fk != map->second.file_keys.end(); ++fk) {
-				auto fk2 = out_entry->second.file_keys.find(fk->first);
+				std::string attribname = fk->first;
+				if (trunc) {
+					attribname = truncate16(attribname, 256);
+				}
+
+				auto fk2 = out_entry->second.file_keys.find(attribname);
 
 				if (fk2 == out_entry->second.file_keys.end()) {
-					out_entry->second.file_keys.insert(*fk);
+					out_entry->second.file_keys.insert(std::pair<std::string, type_and_string_stats>(attribname, fk->second));
 				} else {
 					for (auto val : fk->second.sample_values) {
 						auto pt = std::lower_bound(fk2->second.sample_values.begin(), fk2->second.sample_values.end(), val);

mbtiles.hpp

@@ -46,6 +46,7 @@ void mbtiles_close(sqlite3 *outdb, char **argv);
 void aprintf(std::string *buf, const char *format, ...);
 
 std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps);
+std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps, bool trunc);
 
 void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &layername, type_and_string const &val);
 

tests/longlayer/in.json

@@ -0,0 +1,4 @@
+{ "type": "Feature", "properties": { }, "tippecanoe": { "layer": "ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_1" }, "geometry": { "type": "Point", "coordinates": [ 0,0 ]} }
+{ "type": "Feature", "properties": { }, "tippecanoe": { "layer": "ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_2" }, "geometry": { "type": "Point", "coordinates": [ 0,0 ]} }
+{ "type": "Feature", "properties": { "ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_1": 12 }, "tippecanoe": { }, "geometry": { "type": "Point", "coordinates": [ 0,0 ]} }
+{ "type": "Feature", "properties": { "ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_2": 13 }, "tippecanoe": { }, "geometry": { "type": "Point", "coordinates": [ 0,0 ]} }

tests/longlayer/out/-z0.json

@@ -0,0 +1,29 @@
+{ "type": "FeatureCollection", "properties": {
+"bounds": "0.000000,0.000000,0.000000,0.000000",
+"center": "0.000000,0.000000,0",
+"description": "tests/longlayer/out/-z0.json.check.mbtiles",
+"format": "pbf",
+"json": "{\"vector_layers\": [ { \"id\": \"ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_1\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} }, { \"id\": \"ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_2\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {} }, { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 0, \"fields\": {\"ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_1\": \"Number\", \"ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_2\": \"Number\"} } ] }",
+"maxzoom": "0",
+"minzoom": "0",
+"name": "tests/longlayer/out/-z0.json.check.mbtiles",
+"tilestats": "{\"layerCount\": 2,\"layers\": [{\"layer\": \"ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongN\",\"count\": 2,\"geometry\": \"Point\",\"attributeCount\": 0,\"attributes\": []},{\"layer\": \"in\",\"count\": 2,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHas\",\"count\": 2,\"type\": \"number\",\"values\": [12,13],\"min\": 12.000000,\"max\": 13.000000}]}]}",
+"type": "overlay",
+"version": "2"
+}, "features": [
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_1", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "layer": "ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_ThisLayerHasAVeryLongName_2", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_1": 12 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_ThisAttributeHasAVeryLongName_2": 13 }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+] }
+] }