From 64a5a684b0bd3bcd16716ff4a4f4543e94353f7f Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Wed, 21 Dec 2022 14:57:54 +0800 Subject: [PATCH] tippecanoe-decode reads pmtiles input. [#10] --- Makefile | 2 +- decode.cpp | 87 ++++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 62 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index f65d6863..9133e9fb 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o pro tippecanoe-enumerate: enumerate.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3 -tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o +tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o diff --git a/decode.cpp b/decode.cpp index afec214f..6dec5c8d 100644 --- a/decode.cpp +++ b/decode.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include "mvt.hpp" #include "projection.hpp" #include "geometry.hpp" @@ -23,6 +22,7 @@ #include "jsonpull/jsonpull.h" #include "mbtiles.hpp" #include "dirtiles.hpp" +#include "pmtiles_file.hpp" #include "errors.hpp" int minzoom = 0; @@ -245,7 +245,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co if (st.st_size < 50 * 1024 * 1024) { char *map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (map != NULL && map != MAP_FAILED) { - if (strcmp(map, "SQLite format 3") != 0) { + if (strcmp(map, "SQLite format 3") != 0 && strncmp(map, "PMTiles", 7) != 0) { if (z >= 0) { std::string s = std::string(map, st.st_size); handle(s, z, x, y, to_decode, pipeline, stats, state, coordinate_mode); @@ -272,11 +272,28 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co struct stat st; std::vector tiles; + + char *pmtiles_map; + std::vector entries; + if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) { isdir = true; db = dirmeta2tmp(fname); tiles = enumerate_dirtiles(fname, minzoom, maxzoom); + } else if (pmtiles_has_suffix(fname)) { + int pmtiles_fd = open(fname, O_RDONLY | O_CLOEXEC); + pmtiles_map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, pmtiles_fd, 0); + if (pmtiles_map == MAP_FAILED) { + perror("mmap in decode"); + exit(EXIT_MEMORY); + } + if (close(pmtiles_fd) != 0) { + perror("close"); + exit(EXIT_CLOSE); + } + db = pmtilesmeta2tmp(fname, pmtiles_map); + entries = pmtiles_entries_tms(pmtiles_map, minzoom, maxzoom); } else { if (sqlite3_open(fname, &db) != SQLITE_OK) { fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db)); @@ -382,6 +399,11 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, state, coordinate_mode); } + } else if (entries.size() > 0) { + for (auto const &entry : entries) { + std::string s{pmtiles_map + entry.offset, entry.length}; + handle(s, entry.z, entry.x, entry.y, to_decode, pipeline, stats, state, coordinate_mode); + } } else { const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;"; sqlite3_stmt *stmt; @@ -447,36 +469,49 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co } else { int handled = 0; while (z >= 0 && !handled) { - const char *sql = "SELECT tile_data from tiles where zoom_level = ? and tile_column = ? and tile_row = ?;"; - sqlite3_stmt *stmt; - if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { - fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db)); - exit(EXIT_SQLITE); - } - - sqlite3_bind_int(stmt, 1, z); - sqlite3_bind_int(stmt, 2, x); - sqlite3_bind_int(stmt, 3, (1LL << z) - 1 - y); - - while (sqlite3_step(stmt) == SQLITE_ROW) { - int len = sqlite3_column_bytes(stmt, 0); - const char *s = (const char *) sqlite3_column_blob(stmt, 0); - - if (s == NULL) { - fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n"); + if (entries.size() > 0) { + uint64_t tile_offset; + uint32_t tile_length; + std::tie(tile_offset, tile_length) = pmtiles_get_tile(pmtiles_map, z, x, y); + if (tile_length > 0) { + if (z != oz) { + fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy); + } + std::string s{pmtiles_map + tile_offset, tile_length}; + handle(s, z, x, y, to_decode, pipeline, stats, state, coordinate_mode); + handled = 1; + } + } else { + const char *sql = "SELECT tile_data from tiles where zoom_level = ? and tile_column = ? and tile_row = ?;"; + sqlite3_stmt *stmt; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { + fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_SQLITE); } - if (z != oz) { - fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy); + sqlite3_bind_int(stmt, 1, z); + sqlite3_bind_int(stmt, 2, x); + sqlite3_bind_int(stmt, 3, (1LL << z) - 1 - y); + + while (sqlite3_step(stmt) == SQLITE_ROW) { + int len = sqlite3_column_bytes(stmt, 0); + const char *s = (const char *) sqlite3_column_blob(stmt, 0); + + if (s == NULL) { + fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n"); + exit(EXIT_SQLITE); + } + + if (z != oz) { + fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy); + } + + handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state, coordinate_mode); + handled = 1; } - - handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state, coordinate_mode); - handled = 1; + sqlite3_finalize(stmt); } - sqlite3_finalize(stmt); - z--; x /= 2; y /= 2;