Add a data structure for tileset metadata (#44)

* Progress toward making a tileset metadata structure

* Write metadata from structure to mbtiles

* Write dirtiles metadata.json from metadata structure

* Update changelog

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.15.1
+
+* Correct mbtiles inserts to use text instead of blob
+* Add an internal data structure to represent tileset metadata
+
 ## 2.15.0
 
 * Generate label points in a more straightforward checkerboard, and fewer of them at high zoom levels.

decode.cpp

@@ -21,6 +21,7 @@
 #include "geometry.hpp"
 #include "write_json.hpp"
 #include "jsonpull/jsonpull.h"
+#include "mbtiles.hpp"
 #include "dirtiles.hpp"
 #include "errors.hpp"
 

dirtiles.cpp

@@ -11,8 +11,10 @@
 #include <sys/stat.h>
 #include <sqlite3.h>
 #include "jsonpull/jsonpull.h"
+#include "mbtiles.hpp"
 #include "dirtiles.hpp"
 #include "errors.hpp"
+#include "write_json.hpp"
 
 std::string dir_read_tile(std::string base, struct zxy tile) {
 	std::ifstream pbfFile(base + "/" + tile.path(), std::ios::in | std::ios::binary);
@@ -276,3 +278,73 @@ sqlite3 *dirmeta2tmp(const char *fname) {
 
 	return db;
 }
+
+static void out(json_writer &state, std::string k, std::string v) {
+	state.json_comma_newline();
+	state.json_write_string(k);
+	state.json_write_string(v);
+}
+
+void dir_write_metadata(const char *outdir, const metadata &m) {
+	std::string metadata = std::string(outdir) + "/metadata.json";
+
+	struct stat st;
+	if (stat(metadata.c_str(), &st) == 0) {
+		// Leave existing metadata in place with --allow-existing
+	} else {
+		FILE *fp = fopen(metadata.c_str(), "w");
+		if (fp == NULL) {
+			perror(metadata.c_str());
+			exit(EXIT_OPEN);
+		}
+
+		json_writer state(fp);
+
+		state.json_write_hash();
+		state.json_write_newline();
+
+		out(state, "name", m.name);
+		out(state, "description", m.description);
+		out(state, "version", std::to_string(m.version));
+		out(state, "minzoom", std::to_string(m.minzoom));
+		out(state, "maxzoom", std::to_string(m.maxzoom));
+		out(state, "center", std::to_string(m.center_lon) + "," + std::to_string(m.center_lat) + "," + std::to_string(m.center_z));
+		out(state, "bounds", std::to_string(m.minlon) + "," + std::to_string(m.minlat) + "," +
+                                     std::to_string(m.maxlon) + "," + std::to_string(m.maxlat));
+		out(state, "type", m.type);
+		if (m.attribution.size() > 0) {
+			out(state, "attribution", m.attribution);
+		}
+		if (m.strategies_json.size() > 0) {
+			out(state, "strategies", m.strategies_json);
+		}
+		out(state, "format", m.format);
+		out(state, "generator", m.generator);
+		out(state, "generator_options", m.generator_options);
+
+		if (m.vector_layers_json.size() > 0 || m.tilestats_json.size() > 0) {
+			std::string json = "{";
+
+			if (m.vector_layers_json.size() > 0) {
+				json += "\"vector_layers\": " + m.vector_layers_json;
+
+				if (m.tilestats_json.size() > 0) {
+					json += ",\"tilestats\": " + m.tilestats_json;
+				}
+			} else {
+				if (m.tilestats_json.size() > 0) {
+					json += "\"tilestats\": " + m.tilestats_json;
+				}
+			}
+
+			json += "}";
+
+			out(state, "json", json);
+		}
+
+		state.json_write_newline();
+		state.json_end_hash();
+		state.json_write_newline();
+		fclose(fp);
+	}
+}

dirtiles.hpp

@@ -7,6 +7,7 @@
 
 void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);
 void dir_erase_zoom(const char *outdir, int z);
+void dir_write_metadata(const char *outdir, const metadata &m);
 
 void check_dir(const char *d, char **argv, bool force, bool forcetable);
 

main.cpp

@@ -2564,7 +2564,12 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 		ai->second.maxzoom = maxzoom;
 	}
 
-	mbtiles_write_metadata(outdb, outdir, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies);
+	metadata m = make_metadata(fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies);
+	if (outdb != NULL) {
+		mbtiles_write_metadata(outdb, m, forcetable);
+	} else {
+		dir_write_metadata(outdir, m);
+	}
 
 	return ret;
 }

mbtiles.cpp

@@ -430,22 +430,10 @@ std::string stringify_strategies(std::vector<strategy> const &strategies) {
 	}
 }
 
-void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies) {
+void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable) {
 	char *sql, *err;
 
-	sqlite3 *db = outdb;
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", m.name.c_str());
-	if (outdb == NULL) {
-		if (sqlite3_open("", &db) != SQLITE_OK) {
-			fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db));
-			exit(EXIT_OPEN);
-		}
-		if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) {
-			fprintf(stderr, "Create metadata table: %s\n", err);
-			exit(EXIT_SQLITE);
-		}
-	}
-
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname);
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set name in metadata: %s\n", err);
 		if (!forcetable) {
@@ -454,7 +442,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", description != NULL ? description : fname);
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", m.description.c_str());
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set description in metadata: %s\n", err);
 		if (!forcetable) {
@@ -463,7 +451,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", 2);
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", m.version);
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set version : %s\n", err);
 		if (!forcetable) {
@@ -472,7 +460,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", minzoom);
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", m.minzoom);
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set minzoom: %s\n", err);
 		if (!forcetable) {
@@ -481,7 +469,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", maxzoom);
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", m.maxzoom);
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set maxzoom: %s\n", err);
 		if (!forcetable) {
@@ -490,7 +478,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", midlon, midlat, maxzoom);
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", m.center_lon, m.center_lat, m.center_z);
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set center: %s\n", err);
 		if (!forcetable) {
@@ -499,7 +487,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", minlon, minlat, maxlon, maxlat);
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", m.minlon, m.minlat, m.maxlon, m.maxlat);
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set bounds: %s\n", err);
 		if (!forcetable) {
@@ -508,7 +496,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", "overlay");
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", m.type.c_str());
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set type: %s\n", err);
 		if (!forcetable) {
@@ -517,10 +505,10 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	if (attribution != NULL) {
+	if (m.attribution.size() > 0) {
-		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", attribution);
+		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", m.attribution.c_str());
 		if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
-			fprintf(stderr, "set type: %s\n", err);
+			fprintf(stderr, "set attribution: %s\n", err);
 			if (!forcetable) {
 				exit(EXIT_SQLITE);
 			}
@@ -528,7 +516,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 		sqlite3_free(sql);
 	}
 
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", vector ? "pbf" : "png");
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", m.format.c_str());
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set format: %s\n", err);
 		if (!forcetable) {
@@ -537,17 +525,16 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	std::string version = program + " " + VERSION;
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('generator', %Q);", m.generator.c_str());
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('generator', %Q);", version.c_str());
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
-		fprintf(stderr, "set version: %s\n", err);
+		fprintf(stderr, "set generator: %s\n", err);
 		if (!forcetable) {
 			exit(EXIT_SQLITE);
 		}
 	}
 	sqlite3_free(sql);
 
-	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('generator_options', %Q);", commandline.c_str());
+	sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('generator_options', %Q);", m.generator_options.c_str());
 	if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "set commandline: %s\n", err);
 		if (!forcetable) {
@@ -556,9 +543,8 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 	}
 	sqlite3_free(sql);
 
-	std::string strat = stringify_strategies(strategies);
+	if (m.strategies_json.size() > 0) {
-	if (strat.size() > 0) {
+		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", m.strategies_json.c_str());
-		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", strat.c_str());
 		if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
 			fprintf(stderr, "set strategies: %s\n", err);
 			if (!forcetable) {
@@ -568,17 +554,68 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 		sqlite3_free(sql);
 	}
 
+	if (m.vector_layers_json.size() > 0 || m.tilestats_json.size() > 0) {
+		std::string json = "{";
+
+		if (m.vector_layers_json.size() > 0) {
+			json += "\"vector_layers\": " + m.vector_layers_json;
+
+			if (m.tilestats_json.size() > 0) {
+				json += ",\"tilestats\": " + m.tilestats_json;
+			}
+		} else {
+			if (m.tilestats_json.size() > 0) {
+				json += "\"tilestats\": " + m.tilestats_json;
+			}
+		}
+
+		json += "}";
+
+		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", json.c_str());
+		if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
+			fprintf(stderr, "set json: %s\n", err);
+			if (!forcetable) {
+				exit(EXIT_SQLITE);
+			}
+		}
+		sqlite3_free(sql);
+	}
+}
+
+metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies) {
+	metadata m;
+
+	m.name = fname;
+	m.description = description != NULL ? description : fname;
+	m.version = 2;
+	m.type = "overlay";
+	m.format = vector ? "pbf" : "png";
+
+	m.minzoom = minzoom;
+	m.maxzoom = maxzoom;
+
+	m.minlat = minlat;
+	m.minlon = minlon;
+	m.maxlat = maxlat;
+	m.maxlon = maxlon;
+
+	m.center_lat = midlat;
+	m.center_lon = midlon;
+	m.center_z = maxzoom;
+
+	if (attribution != NULL) {
+		m.attribution = attribution;
+	}
+
+	m.generator = program + " " + VERSION;
+	m.generator_options = commandline;
+
+	m.strategies_json = stringify_strategies(strategies);
+
 	if (vector) {
-		size_t elements = max_tilestats_values;
-		std::string buf;
-
 		{
-			json_writer state(&buf);
+			json_writer state(&m.vector_layers_json);
 
-			state.json_write_hash();
-			state.nospace = true;
-
-			state.json_write_string("vector_layers");
 			state.json_write_array();
 
 			std::vector<std::string> lnames;
@@ -647,77 +684,19 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 			}
 
 			state.json_end_array();
+		}
+
+		{
+			size_t elements = max_tilestats_values;
+			json_writer state(&m.tilestats_json);
 
 			if (do_tilestats && elements > 0) {
-				state.nospace = true;
-				state.json_write_string("tilestats");
 				tilestats(layermap, elements, state);
 			}
-
-			state.nospace = true;
-			state.json_end_hash();
-		}
-
-		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str());
-		if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
-			fprintf(stderr, "set json: %s\n", err);
-			if (!forcetable) {
-				exit(EXIT_SQLITE);
-			}
-		}
-		sqlite3_free(sql);
-	}
-
-	if (outdir != NULL) {
-		std::string metadata = std::string(outdir) + "/metadata.json";
-
-		struct stat st;
-		if (stat(metadata.c_str(), &st) == 0) {
-			// Leave existing metadata in place with --allow-existing
-		} else {
-			FILE *fp = fopen(metadata.c_str(), "w");
-			if (fp == NULL) {
-				perror(metadata.c_str());
-				exit(EXIT_OPEN);
-			}
-
-			json_writer state(fp);
-
-			state.json_write_hash();
-			state.json_write_newline();
-
-			sqlite3_stmt *stmt;
-			if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) {
-				while (sqlite3_step(stmt) == SQLITE_ROW) {
-					std::string key, value;
-
-					const char *k = (const char *) sqlite3_column_text(stmt, 0);
-					const char *v = (const char *) sqlite3_column_text(stmt, 1);
-					if (k == NULL || v == NULL) {
-						fprintf(stderr, "Corrupt mbtiles file: null metadata\n");
-						exit(EXIT_SQLITE);
-					}
-
-					state.json_comma_newline();
-					state.json_write_string(k);
-					state.json_write_string(v);
-				}
-				sqlite3_finalize(stmt);
-			}
-
-			state.json_write_newline();
-			state.json_end_hash();
-			state.json_write_newline();
-			fclose(fp);
 		}
 	}
 
-	if (outdb == NULL) {
+	return m;
-		if (sqlite3_close(db) != SQLITE_OK) {
-			fprintf(stderr, "Could not close temp database: %s\n", sqlite3_errmsg(db));
-			exit(EXIT_CLOSE);
-		}
-	}
 }
 
 void mbtiles_close(sqlite3 *outdb, const char *pgm) {

mbtiles.hpp

@@ -41,6 +41,32 @@ struct layermap_entry {
 	}
 };
 
+struct metadata {
+	std::string name;
+	std::string description;
+	int version;
+	std::string type;
+	std::string format;
+
+	int minzoom;
+	int maxzoom;
+
+	double minlat, minlon, maxlat, maxlon;
+
+	double center_lon, center_lat;
+	int center_z;
+
+	std::string attribution; // not written if empty
+
+	std::string generator;
+	std::string generator_options;
+
+	std::string strategies_json; // not written if empty
+
+	std::string vector_layers_json;
+	std::string tilestats_json;
+};
+
 #include "tile.hpp"
 
 sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
@@ -48,7 +74,8 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
 void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
 void mbtiles_erase_zoom(sqlite3 *outdb, int z);
 
-void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies);
+metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies);
+void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable);
 
 void mbtiles_close(sqlite3 *outdb, const char *pgm);
 

tile-join.cpp

@@ -1204,7 +1204,13 @@ int main(int argc, char **argv) {
 		}
 	}
 
-	mbtiles_write_metadata(outdb, out_dir, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options, strategies);
+	metadata m = make_metadata(name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options, strategies);
+
+	if (outdb != NULL) {
+		mbtiles_write_metadata(outdb, m, true);
+	} else {
+		dir_write_metadata(out_dir, m);
+	}
 
 	if (outdb != NULL) {
 		mbtiles_close(outdb, argv[0]);

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.15.0"
+#define VERSION "v2.15.1"
 
 #endif