Fix tilestats with truncated strings. Trim tilestats if TileJSON is huge

mbtiles.cpp

@@ -137,7 +137,7 @@ bool type_and_string::operator!=(const type_and_string &o) const {
 	return false;
 }
 
-std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
+std::string tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t elements) {
 	// Consolidate layers/attributes whose names are truncated
 	std::vector<std::map<std::string, layermap_entry>> lmv;
 	lmv.push_back(layermap1);
@@ -192,7 +192,7 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
 
 		size_t attrs = 0;
 		for (auto attribute : layer.second.file_keys) {
-			if (attrs == 100) {
+			if (attrs == elements) {
 				break;
 			}
 			if (attrs != 0) {
@@ -240,21 +240,27 @@ std::string tilestats(std::map<std::string, layermap_entry> const &layermap1) {
 
 			size_t vals = 0;
 			for (auto value : attribute.second.sample_values) {
-				if (vals == 100) {
+				if (vals == elements) {
 					break;
 				}
-				if (vals != 0) {
-					out.append(",\n");
-				}
-				vals++;
 
 				if (value.type == mvt_double || value.type == mvt_bool) {
+					if (vals != 0) {
+						out.append(",\n");
+					}
+					vals++;
+
 					out.append("\t\t\t\t\t\t");
 					out.append(value.string);
 				} else {
 					std::string trunc = truncate16(value.string, 256);
 
 					if (trunc.size() == value.string.size()) {
+						if (vals != 0) {
+							out.append(",\n");
+						}
+						vals++;
+
 						out.append("\t\t\t\t\t\t\"");
 						quote(out, value.string.c_str());
 						out.append("\"");
@@ -407,62 +413,74 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	sqlite3_free(sql);
 
 	if (vector) {
-		std::string buf("{");
+		size_t elements = 100;
-		aprintf(&buf, "\"vector_layers\": [ ");
+		std::string buf;
 
-		std::vector<std::string> lnames;
+		while (1) {
-		for (auto ai = layermap.begin(); ai != layermap.end(); ++ai) {
+			buf = "{";
-			lnames.push_back(ai->first);
+			aprintf(&buf, "\"vector_layers\": [ ");
-		}
 
-		for (size_t i = 0; i < lnames.size(); i++) {
+			std::vector<std::string> lnames;
-			if (i != 0) {
+			for (auto ai = layermap.begin(); ai != layermap.end(); ++ai) {
-				aprintf(&buf, ", ");
+				lnames.push_back(ai->first);
 			}
 
-			auto fk = layermap.find(lnames[i]);
+			for (size_t i = 0; i < lnames.size(); i++) {
-			aprintf(&buf, "{ \"id\": \"");
+				if (i != 0) {
-			quote(buf, lnames[i]);
-			aprintf(&buf, "\", \"description\": \"\", \"minzoom\": %d, \"maxzoom\": %d, \"fields\": {", fk->second.minzoom, fk->second.maxzoom);
-
-			bool first = true;
-			for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) {
-				if (first) {
-					first = false;
-				} else {
 					aprintf(&buf, ", ");
 				}
 
-				aprintf(&buf, "\"");
+				auto fk = layermap.find(lnames[i]);
-				quote(buf, j->first.c_str());
+				aprintf(&buf, "{ \"id\": \"");
+				quote(buf, lnames[i]);
+				aprintf(&buf, "\", \"description\": \"\", \"minzoom\": %d, \"maxzoom\": %d, \"fields\": {", fk->second.minzoom, fk->second.maxzoom);
 
-				int type = 0;
+				bool first = true;
-				for (auto s : j->second.sample_values) {
+				for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) {
-					type |= (1 << s.type);
+					if (first) {
+						first = false;
+					} else {
+						aprintf(&buf, ", ");
+					}
+
+					aprintf(&buf, "\"");
+					quote(buf, j->first.c_str());
+
+					int type = 0;
+					for (auto s : j->second.sample_values) {
+						type |= (1 << s.type);
+					}
+
+					if (type == (1 << mvt_double)) {
+						aprintf(&buf, "\": \"Number\"");
+					} else if (type == (1 << mvt_bool)) {
+						aprintf(&buf, "\": \"Boolean\"");
+					} else if (type == (1 << mvt_string)) {
+						aprintf(&buf, "\": \"String\"");
+					} else {
+						aprintf(&buf, "\": \"Mixed\"");
+					}
 				}
 
-				if (type == (1 << mvt_double)) {
+				aprintf(&buf, "} }");
-					aprintf(&buf, "\": \"Number\"");
-				} else if (type == (1 << mvt_bool)) {
-					aprintf(&buf, "\": \"Boolean\"");
-				} else if (type == (1 << mvt_string)) {
-					aprintf(&buf, "\": \"String\"");
-				} else {
-					aprintf(&buf, "\": \"Mixed\"");
-				}
 			}
 
-			aprintf(&buf, "} }");
+			aprintf(&buf, " ]");
+
+			if (do_tilestats && elements > 0) {
+				aprintf(&buf, ",\"tilestats\": %s", tilestats(layermap, elements).c_str());
+			}
+
+			aprintf(&buf, "}");
+
+			if (buf.size() >= 60 * 1024 && elements > 0) {
+				elements = elements / 2;
+				continue;
+			} else {
+				break;
+			}
 		}
 
-		aprintf(&buf, " ]");
-
-		if (do_tilestats) {
-			aprintf(&buf, ",\"tilestats\": %s", tilestats(layermap).c_str());
-		}
-
-		aprintf(&buf, "}");
-
 		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str());
 		if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 			fprintf(stderr, "set json: %s\n", err);