#include #include #include #include #include #include #include #include #include "errors.hpp" #include "pmtiles_file.hpp" #include "mvt.hpp" #include "write_json.hpp" #include "main.hpp" bool pmtiles_has_suffix(const char *filename) { if (filename == nullptr) { return false; } size_t lenstr = strlen(filename); if (lenstr < 8) { return false; } if (strncmp(filename + (lenstr - 8), ".pmtiles", 8) == 0) { return true; } return false; } void check_pmtiles(const char *filename, char **argv, bool forcetable) { struct stat st; if (stat(filename, &st) == 0) { fprintf(stderr, "%s: Tileset \"%s\" already exists. You can use --force if you want to delete the old tileset.\n", argv[0], filename); fprintf(stderr, "%s: %s: file exists\n", argv[0], filename); if (forcetable) { fprintf(stderr, "%s: --allow-existing is not supported for pmtiles\n", argv[0]); } exit(EXIT_EXISTS); } } std::string decompress_fn(const std::string &input, uint8_t compression) { std::string output; if (compression == pmtiles::COMPRESSION_NONE) { output = input; } else if (compression == pmtiles::COMPRESSION_GZIP) { decompress(input, output); } else { throw std::runtime_error("Unknown or unsupported compression."); } return output; } std::string compress_fn(const std::string &input, uint8_t compression) { std::string output; if (compression == pmtiles::COMPRESSION_NONE) { output = input; } else if (compression == pmtiles::COMPRESSION_GZIP) { compress(input, output, true); } else { throw std::runtime_error("Unknown or unsupported compression."); } return output; } std::vector pmtiles_entries_tms(const char *pmtiles_map, int minzoom, int maxzoom) { std::vector filtered; auto all_entries = pmtiles::entries_tms(&decompress_fn, pmtiles_map); std::copy_if(all_entries.begin(), all_entries.end(), std::back_inserter(filtered), [minzoom, maxzoom](pmtiles::entry_zxy e) { return e.z >= minzoom && e.z <= maxzoom; }); return filtered; } std::pair pmtiles_get_tile(const char *pmtiles_map, int z, int x, int y) { return pmtiles::get_tile(&decompress_fn, pmtiles_map, z, x, y); } static void out(json_writer &state, std::string k, std::string v) { state.json_comma_newline(); state.json_write_string(k); state.json_write_string(v); } std::string metadata_to_pmtiles_json(metadata m) { std::string buf; json_writer state(&buf); state.json_write_hash(); state.json_write_newline(); out(state, "name", m.name); out(state, "format", m.format); out(state, "type", m.type); out(state, "description", m.description); out(state, "version", std::to_string(m.version)); if (m.attribution.size() > 0) { out(state, "attribution", m.attribution); } if (m.strategies_json.size() > 0) { state.json_comma_newline(); state.json_write_string("strategies"); state.json_write_json(m.strategies_json); } if (m.decisions_json.size() > 0) { state.json_comma_newline(); state.json_write_string("tippecanoe_decisions"); state.json_write_json(m.decisions_json); } out(state, "generator", m.generator); out(state, "generator_options", m.generator_options); std::string bounds2 = std::to_string(m.minlon2) + "," + std::to_string(m.minlat2) + "," + std::to_string(m.maxlon2) + "," + std::to_string(m.maxlat2); out(state, "antimeridian_adjusted_bounds", bounds2); if (m.vector_layers_json.size() > 0) { state.json_comma_newline(); state.json_write_string("vector_layers"); state.json_write_json(m.vector_layers_json); } if (m.tilestats_json.size() > 0) { state.json_comma_newline(); state.json_write_string("tilestats"); state.json_write_json(m.tilestats_json); } state.json_write_newline(); state.json_end_hash(); state.json_write_newline(); std::string compressed; compress(buf, compressed, true); return compressed; } void mbtiles_map_image_to_pmtiles(char *fname, metadata m, bool tile_compression, bool fquiet, bool fquiet_progress) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_SQLITE); } char *err = NULL; if (sqlite3_exec(db, "PRAGMA integrity_check;", NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "%s: integrity_check: %s\n", fname, err); exit(EXIT_SQLITE); } // materialize list of all tile IDs std::vector tile_ids; { const char *sql = "SELECT zoom_level, tile_column, tile_row FROM map"; sqlite3_stmt *stmt; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_SQLITE); } while (sqlite3_step(stmt) == SQLITE_ROW) { int zoom = sqlite3_column_int(stmt, 0); int x = sqlite3_column_int(stmt, 1); int sorty = sqlite3_column_int(stmt, 2); int y = (1LL << zoom) - 1 - sorty; uint64_t res = pmtiles::zxy_to_tileid(zoom, x, y); tile_ids.push_back(res); } sqlite3_finalize(stmt); } std::stable_sort(tile_ids.begin(), tile_ids.end()); std::unordered_map> hash_to_offset_len; std::vector entries; unsigned long long offset = 0; std::string tmpname = (std::string(fname) + ".tmp"); // write tile data to tempfile in clustered order { const char *map_sql = "SELECT tile_id FROM map WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?"; sqlite3_stmt *map_stmt; if (sqlite3_prepare_v2(db, map_sql, -1, &map_stmt, NULL) != SQLITE_OK) { fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_SQLITE); } const char *image_sql = "SELECT tile_data FROM images WHERE tile_id = ?"; sqlite3_stmt *image_stmt; if (sqlite3_prepare_v2(db, image_sql, -1, &image_stmt, NULL) != SQLITE_OK) { fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_SQLITE); } std::ofstream tmp_ostream; tmp_ostream.open(tmpname.c_str(), std::ios::out | std::ios::binary); int idx = 0; int progress_reported = -1; for (auto const &tile_id : tile_ids) { idx = idx + 1; double progress = ((double) idx / tile_ids.size()) * 100; pmtiles::zxy zxy = pmtiles::tileid_to_zxy(tile_id); if (!fquiet && !fquiet_progress && progress_time() && (int) progress != progress_reported) { fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, zxy.z, zxy.x, zxy.y); progress_reported = (int) progress; fflush(stderr); } sqlite3_bind_int(map_stmt, 1, zxy.z); sqlite3_bind_int(map_stmt, 2, zxy.x); sqlite3_bind_int(map_stmt, 3, (1LL << zxy.z) - 1 - zxy.y); if (sqlite3_step(map_stmt) != SQLITE_ROW) { fprintf(stderr, "Corrupt mbtiles file: null entry in map table\n"); exit(EXIT_SQLITE); } std::string hsh{reinterpret_cast(sqlite3_column_text(map_stmt, 0))}; if (hash_to_offset_len.count(hsh) > 0) { auto offset_len = hash_to_offset_len.at(hsh); if (entries.size() > 0 && tile_id == entries[entries.size() - 1].tile_id + 1 && entries[entries.size() - 1].offset == std::get<0>(offset_len)) { entries[entries.size() - 1].run_length++; } else { entries.emplace_back(tile_id, std::get<0>(offset_len), std::get<1>(offset_len), 1); } } else { sqlite3_bind_text(image_stmt, 1, hsh.data(), hsh.size(), SQLITE_STATIC); if (sqlite3_step(image_stmt) != SQLITE_ROW) { fprintf(stderr, "Corrupt mbtiles file: null entry in image table\n"); exit(EXIT_SQLITE); } int len = sqlite3_column_bytes(image_stmt, 0); const char *blob = (const char *) sqlite3_column_blob(image_stmt, 0); tmp_ostream.write(blob, len); entries.emplace_back(tile_id, offset, len, 1); hash_to_offset_len.emplace(hsh, std::make_pair(offset, len)); offset += len; sqlite3_reset(image_stmt); sqlite3_clear_bindings(image_stmt); } sqlite3_reset(map_stmt); sqlite3_clear_bindings(map_stmt); } tmp_ostream.close(); sqlite3_finalize(map_stmt); sqlite3_finalize(image_stmt); } // finalize PMTiles archive. { std::stable_sort(entries.begin(), entries.end(), pmtiles::entryv3_cmp()); std::string root_bytes; std::string leaves_bytes; int num_leaves; std::tie(root_bytes, leaves_bytes, num_leaves) = make_root_leaves(&compress_fn, pmtiles::COMPRESSION_GZIP, entries); pmtiles::headerv3 header; header.min_zoom = m.minzoom; header.max_zoom = m.maxzoom; header.min_lon_e7 = m.minlon * 10000000; header.min_lat_e7 = m.minlat * 10000000; header.max_lon_e7 = m.maxlon * 10000000; header.max_lat_e7 = m.maxlat * 10000000; header.center_zoom = m.center_z; header.center_lon_e7 = m.center_lon * 10000000; header.center_lat_e7 = m.center_lat * 10000000; std::string json_metadata = metadata_to_pmtiles_json(m); sqlite3_close(db); header.clustered = 0x1; header.internal_compression = pmtiles::COMPRESSION_GZIP; if (tile_compression) { header.tile_compression = pmtiles::COMPRESSION_GZIP; } else { header.tile_compression = pmtiles::COMPRESSION_NONE; } if (m.format == "pbf") { header.tile_type = pmtiles::TILETYPE_MVT; } else if (m.format == "png") { header.tile_type = pmtiles::TILETYPE_PNG; } else { header.tile_type = pmtiles::TILETYPE_UNKNOWN; } header.root_dir_offset = 127; header.root_dir_bytes = root_bytes.size(); header.json_metadata_offset = header.root_dir_offset + header.root_dir_bytes; header.json_metadata_bytes = json_metadata.size(); header.leaf_dirs_offset = header.json_metadata_offset + header.json_metadata_bytes; header.leaf_dirs_bytes = leaves_bytes.size(); header.tile_data_offset = header.leaf_dirs_offset + header.leaf_dirs_bytes; header.tile_data_bytes = offset; header.addressed_tiles_count = tile_ids.size(); header.tile_entries_count = entries.size(); header.tile_contents_count = hash_to_offset_len.size(); std::ifstream tmp_istream(tmpname.c_str(), std::ios::in | std::ios_base::binary); std::ofstream ostream; ostream.open(fname, std::ios::out | std::ios::binary); auto header_str = header.serialize(); ostream.write(header_str.data(), header_str.length()); ostream.write(root_bytes.data(), root_bytes.length()); ostream.write(json_metadata.data(), json_metadata.size()); ostream.write(leaves_bytes.data(), leaves_bytes.length()); ostream << tmp_istream.rdbuf(); tmp_istream.close(); unlink(tmpname.c_str()); ostream.close(); } } // this should go away if we get rid of temporary metadata DBs. // it transforms the PMTiles header + json into string keys/string values // consistent with mbtiles and dirtiles, for the test suite sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { sqlite3 *db; char *sql; char *err; if (sqlite3_open("", &db) != SQLITE_OK) { fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db)); exit(EXIT_SQLITE); } if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "Create metadata table: %s\n", err); exit(EXIT_SQLITE); } std::string header_s{pmtiles_map, 127}; auto header = pmtiles::deserialize_header(header_s); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", header.min_zoom); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set minzoom: %s\n", err); } sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", header.max_zoom); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set maxzoom: %s\n", err); } sqlite3_free(sql); double center_lon = double(header.center_lon_e7) / 10000000.0; double center_lat = double(header.center_lat_e7) / 10000000.0; sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", center_lon, center_lat, header.center_zoom); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set center: %s\n", err); } sqlite3_free(sql); double minlon = double(header.min_lon_e7) / 10000000.0; double minlat = double(header.min_lat_e7) / 10000000.0; double maxlon = double(header.max_lon_e7) / 10000000.0; double maxlat = double(header.max_lat_e7) / 10000000.0; sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", minlon, minlat, maxlon, maxlat); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set bounds: %s\n", err); } sqlite3_free(sql); std::string json_s{pmtiles_map + header.json_metadata_offset, header.json_metadata_bytes}; std::string decompressed_json; if (header.internal_compression == pmtiles::COMPRESSION_NONE) { decompressed_json = json_s; } else if (header.internal_compression == pmtiles::COMPRESSION_GZIP) { decompress(json_s, decompressed_json); } else { fprintf(stderr, "Unknown or unsupported pmtiles compression: %d\n", header.internal_compression); exit(EXIT_OPEN); } json_pull_ptr jp = json_begin_string(decompressed_json.c_str()); json_object_ptr o = json_read_tree(jp); if (o == nullptr) { fprintf(stderr, "%s: metadata parsing error: %s\n", fname, jp->error); exit(EXIT_JSON); } if (o->type != JSON_HASH) { fprintf(stderr, "%s: bad metadata format\n", fname); exit(EXIT_JSON); } bool has_json = false; std::string buf; json_writer state(&buf); state.nospace = true; state.json_write_hash(); for (const auto &e : o->entries()) { // Check before reading, since string() asserts on the type. if (e.key->type != JSON_STRING) { fprintf(stderr, "%s: non-string key in metadata\n", fname); continue; } const std::string &key = e.key->string(); if (key == "vector_layers" && e.value->type == JSON_ARRAY) { has_json = true; state.nospace = true; state.json_write_string("vector_layers"); state.nospace = true; state.json_write_json(json_stringify(e.value.get())); } else if (key == "tilestats" && e.value->type == JSON_HASH) { has_json = true; state.nospace = true; state.json_write_string("tilestats"); state.nospace = true; state.json_write_json(json_stringify(e.value.get())); } else if (key == "strategies" && e.value->type == JSON_ARRAY) { sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(e.value.get()).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); } else if (key == "tippecanoe_decisions" && e.value->type == JSON_HASH) { sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(e.value.get()).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); } else if (e.value->type != JSON_STRING) { fprintf(stderr, "%s\n", key.c_str()); fprintf(stderr, "%s: non-string in metadata\n", fname); } else { sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key.c_str(), e.value->string().c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); } } state.nospace = true; state.json_end_hash(); if (has_json) { sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set json in metadata: %s\n", err); } sqlite3_free(sql); } return db; }