diff --git a/CHANGELOG.md b/CHANGELOG.md index 67067259..20286a02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 2.15.1 + +* Correct mbtiles inserts to use text instead of blob +* Add an internal data structure to represent tileset metadata + ## 2.15.0 * Generate label points in a more straightforward checkerboard, and fewer of them at high zoom levels. diff --git a/decode.cpp b/decode.cpp index 9a7869b2..afec214f 100644 --- a/decode.cpp +++ b/decode.cpp @@ -21,6 +21,7 @@ #include "geometry.hpp" #include "write_json.hpp" #include "jsonpull/jsonpull.h" +#include "mbtiles.hpp" #include "dirtiles.hpp" #include "errors.hpp" diff --git a/dirtiles.cpp b/dirtiles.cpp index 7e293b99..19c6805b 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -11,8 +11,10 @@ #include #include #include "jsonpull/jsonpull.h" +#include "mbtiles.hpp" #include "dirtiles.hpp" #include "errors.hpp" +#include "write_json.hpp" std::string dir_read_tile(std::string base, struct zxy tile) { std::ifstream pbfFile(base + "/" + tile.path(), std::ios::in | std::ios::binary); @@ -276,3 +278,73 @@ sqlite3 *dirmeta2tmp(const char *fname) { return db; } + +static void out(json_writer &state, std::string k, std::string v) { + state.json_comma_newline(); + state.json_write_string(k); + state.json_write_string(v); +} + +void dir_write_metadata(const char *outdir, const metadata &m) { + std::string metadata = std::string(outdir) + "/metadata.json"; + + struct stat st; + if (stat(metadata.c_str(), &st) == 0) { + // Leave existing metadata in place with --allow-existing + } else { + FILE *fp = fopen(metadata.c_str(), "w"); + if (fp == NULL) { + perror(metadata.c_str()); + exit(EXIT_OPEN); + } + + json_writer state(fp); + + state.json_write_hash(); + state.json_write_newline(); + + out(state, "name", m.name); + out(state, "description", m.description); + out(state, "version", std::to_string(m.version)); + out(state, "minzoom", std::to_string(m.minzoom)); + out(state, "maxzoom", std::to_string(m.maxzoom)); + out(state, "center", std::to_string(m.center_lon) + "," + std::to_string(m.center_lat) + "," + std::to_string(m.center_z)); + out(state, "bounds", std::to_string(m.minlon) + "," + std::to_string(m.minlat) + "," + + std::to_string(m.maxlon) + "," + std::to_string(m.maxlat)); + out(state, "type", m.type); + if (m.attribution.size() > 0) { + out(state, "attribution", m.attribution); + } + if (m.strategies_json.size() > 0) { + out(state, "strategies", m.strategies_json); + } + out(state, "format", m.format); + out(state, "generator", m.generator); + out(state, "generator_options", m.generator_options); + + if (m.vector_layers_json.size() > 0 || m.tilestats_json.size() > 0) { + std::string json = "{"; + + if (m.vector_layers_json.size() > 0) { + json += "\"vector_layers\": " + m.vector_layers_json; + + if (m.tilestats_json.size() > 0) { + json += ",\"tilestats\": " + m.tilestats_json; + } + } else { + if (m.tilestats_json.size() > 0) { + json += "\"tilestats\": " + m.tilestats_json; + } + } + + json += "}"; + + out(state, "json", json); + } + + state.json_write_newline(); + state.json_end_hash(); + state.json_write_newline(); + fclose(fp); + } +} diff --git a/dirtiles.hpp b/dirtiles.hpp index 83149cb6..099fa4f0 100644 --- a/dirtiles.hpp +++ b/dirtiles.hpp @@ -7,6 +7,7 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf); void dir_erase_zoom(const char *outdir, int z); +void dir_write_metadata(const char *outdir, const metadata &m); void check_dir(const char *d, char **argv, bool force, bool forcetable); diff --git a/main.cpp b/main.cpp index 7f5dc54c..665a920f 100644 --- a/main.cpp +++ b/main.cpp @@ -2564,7 +2564,12 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo ai->second.maxzoom = maxzoom; } - mbtiles_write_metadata(outdb, outdir, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies); + metadata m = make_metadata(fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies); + if (outdb != NULL) { + mbtiles_write_metadata(outdb, m, forcetable); + } else { + dir_write_metadata(outdir, m); + } return ret; } diff --git a/mbtiles.cpp b/mbtiles.cpp index 9ec71b12..01df016f 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -430,22 +430,10 @@ std::string stringify_strategies(std::vector const &strategies) { } } -void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description, bool do_tilestats, std::map const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector const &strategies) { +void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable) { char *sql, *err; - sqlite3 *db = outdb; - if (outdb == NULL) { - if (sqlite3_open("", &db) != SQLITE_OK) { - fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db)); - exit(EXIT_OPEN); - } - if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "Create metadata table: %s\n", err); - exit(EXIT_SQLITE); - } - } - - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", m.name.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set name in metadata: %s\n", err); if (!forcetable) { @@ -454,7 +442,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", description != NULL ? description : fname); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", m.description.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set description in metadata: %s\n", err); if (!forcetable) { @@ -463,7 +451,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", 2); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", m.version); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set version : %s\n", err); if (!forcetable) { @@ -472,7 +460,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", minzoom); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", m.minzoom); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set minzoom: %s\n", err); if (!forcetable) { @@ -481,7 +469,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", maxzoom); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", m.maxzoom); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set maxzoom: %s\n", err); if (!forcetable) { @@ -490,7 +478,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", midlon, midlat, maxzoom); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", m.center_lon, m.center_lat, m.center_z); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set center: %s\n", err); if (!forcetable) { @@ -499,7 +487,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", minlon, minlat, maxlon, maxlat); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", m.minlon, m.minlat, m.maxlon, m.maxlat); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set bounds: %s\n", err); if (!forcetable) { @@ -508,7 +496,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", "overlay"); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", m.type.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set type: %s\n", err); if (!forcetable) { @@ -517,10 +505,10 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - if (attribution != NULL) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", attribution); + if (m.attribution.size() > 0) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", m.attribution.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set type: %s\n", err); + fprintf(stderr, "set attribution: %s\n", err); if (!forcetable) { exit(EXIT_SQLITE); } @@ -528,7 +516,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam sqlite3_free(sql); } - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", vector ? "pbf" : "png"); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", m.format.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set format: %s\n", err); if (!forcetable) { @@ -537,17 +525,16 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - std::string version = program + " " + VERSION; - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('generator', %Q);", version.c_str()); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('generator', %Q);", m.generator.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set version: %s\n", err); + fprintf(stderr, "set generator: %s\n", err); if (!forcetable) { exit(EXIT_SQLITE); } } sqlite3_free(sql); - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('generator_options', %Q);", commandline.c_str()); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('generator_options', %Q);", m.generator_options.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set commandline: %s\n", err); if (!forcetable) { @@ -556,9 +543,8 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } sqlite3_free(sql); - std::string strat = stringify_strategies(strategies); - if (strat.size() > 0) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", strat.c_str()); + if (m.strategies_json.size() > 0) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", m.strategies_json.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set strategies: %s\n", err); if (!forcetable) { @@ -568,17 +554,68 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam sqlite3_free(sql); } + if (m.vector_layers_json.size() > 0 || m.tilestats_json.size() > 0) { + std::string json = "{"; + + if (m.vector_layers_json.size() > 0) { + json += "\"vector_layers\": " + m.vector_layers_json; + + if (m.tilestats_json.size() > 0) { + json += ",\"tilestats\": " + m.tilestats_json; + } + } else { + if (m.tilestats_json.size() > 0) { + json += "\"tilestats\": " + m.tilestats_json; + } + } + + json += "}"; + + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", json.c_str()); + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "set json: %s\n", err); + if (!forcetable) { + exit(EXIT_SQLITE); + } + } + sqlite3_free(sql); + } +} + +metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, const char *attribution, std::map const &layermap, bool vector, const char *description, bool do_tilestats, std::map const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector const &strategies) { + metadata m; + + m.name = fname; + m.description = description != NULL ? description : fname; + m.version = 2; + m.type = "overlay"; + m.format = vector ? "pbf" : "png"; + + m.minzoom = minzoom; + m.maxzoom = maxzoom; + + m.minlat = minlat; + m.minlon = minlon; + m.maxlat = maxlat; + m.maxlon = maxlon; + + m.center_lat = midlat; + m.center_lon = midlon; + m.center_z = maxzoom; + + if (attribution != NULL) { + m.attribution = attribution; + } + + m.generator = program + " " + VERSION; + m.generator_options = commandline; + + m.strategies_json = stringify_strategies(strategies); + if (vector) { - size_t elements = max_tilestats_values; - std::string buf; - { - json_writer state(&buf); + json_writer state(&m.vector_layers_json); - state.json_write_hash(); - state.nospace = true; - - state.json_write_string("vector_layers"); state.json_write_array(); std::vector lnames; @@ -647,77 +684,19 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam } state.json_end_array(); + } + + { + size_t elements = max_tilestats_values; + json_writer state(&m.tilestats_json); if (do_tilestats && elements > 0) { - state.nospace = true; - state.json_write_string("tilestats"); tilestats(layermap, elements, state); } - - state.nospace = true; - state.json_end_hash(); - } - - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str()); - if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set json: %s\n", err); - if (!forcetable) { - exit(EXIT_SQLITE); - } - } - sqlite3_free(sql); - } - - if (outdir != NULL) { - std::string metadata = std::string(outdir) + "/metadata.json"; - - struct stat st; - if (stat(metadata.c_str(), &st) == 0) { - // Leave existing metadata in place with --allow-existing - } else { - FILE *fp = fopen(metadata.c_str(), "w"); - if (fp == NULL) { - perror(metadata.c_str()); - exit(EXIT_OPEN); - } - - json_writer state(fp); - - state.json_write_hash(); - state.json_write_newline(); - - sqlite3_stmt *stmt; - if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) { - while (sqlite3_step(stmt) == SQLITE_ROW) { - std::string key, value; - - const char *k = (const char *) sqlite3_column_text(stmt, 0); - const char *v = (const char *) sqlite3_column_text(stmt, 1); - if (k == NULL || v == NULL) { - fprintf(stderr, "Corrupt mbtiles file: null metadata\n"); - exit(EXIT_SQLITE); - } - - state.json_comma_newline(); - state.json_write_string(k); - state.json_write_string(v); - } - sqlite3_finalize(stmt); - } - - state.json_write_newline(); - state.json_end_hash(); - state.json_write_newline(); - fclose(fp); } } - if (outdb == NULL) { - if (sqlite3_close(db) != SQLITE_OK) { - fprintf(stderr, "Could not close temp database: %s\n", sqlite3_errmsg(db)); - exit(EXIT_CLOSE); - } - } + return m; } void mbtiles_close(sqlite3 *outdb, const char *pgm) { diff --git a/mbtiles.hpp b/mbtiles.hpp index 8fcd17d6..e7a4ac4d 100644 --- a/mbtiles.hpp +++ b/mbtiles.hpp @@ -41,6 +41,32 @@ struct layermap_entry { } }; +struct metadata { + std::string name; + std::string description; + int version; + std::string type; + std::string format; + + int minzoom; + int maxzoom; + + double minlat, minlon, maxlat, maxlon; + + double center_lon, center_lat; + int center_z; + + std::string attribution; // not written if empty + + std::string generator; + std::string generator_options; + + std::string strategies_json; // not written if empty + + std::string vector_layers_json; + std::string tilestats_json; +}; + #include "tile.hpp" sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); @@ -48,7 +74,8 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); void mbtiles_erase_zoom(sqlite3 *outdb, int z); -void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description, bool do_tilestats, std::map const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector const &strategies); +metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, const char *attribution, std::map const &layermap, bool vector, const char *description, bool do_tilestats, std::map const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector const &strategies); +void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable); void mbtiles_close(sqlite3 *outdb, const char *pgm); diff --git a/tile-join.cpp b/tile-join.cpp index efa1fe83..4814ea7d 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -1204,7 +1204,13 @@ int main(int argc, char **argv) { } } - mbtiles_write_metadata(outdb, out_dir, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options, strategies); + metadata m = make_metadata(name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options, strategies); + + if (outdb != NULL) { + mbtiles_write_metadata(outdb, m, true); + } else { + dir_write_metadata(out_dir, m); + } if (outdb != NULL) { mbtiles_close(outdb, argv[0]); diff --git a/version.hpp b/version.hpp index 8bba085b..9b4fc0d1 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v2.15.0" +#define VERSION "v2.15.1" #endif