diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add support for pmtiles output format (#55)

Browse Source 
* add pmtiles.hpp from github.com/protomaps/PMTiles [#10]

* tippecanoe main writes pmtiles output. [#10]

* detect output format using suffix
* after mbtiles is done writing, replace with pmtiles based on map/image tables.
* add method to write_json for writing json sub-object.

* tippecanoe-decode reads pmtiles input. [#10]

* tile-join reads and writes pmtiles. [#10]

* pmtiles test suite for decode and tile-join [#10]

* add base GitHub CI action for compiling and test suite.

* update pmtiles.hpp with z>15 fix

* Fix some ordering problems with pmtiles decode

* Pmtiles should also pass the raw tiles tests

* Eradicate spaces from tileset metadata JSON fields

* Eradicate spaces from more test fixtures

* Update more tests

* Pmtiles tests pass now too

* Remove unnecessary sort (and make indent)

* Update changelog

* The allow-existing test for pmtiles needs -o, not -e

* Declare --allow-existing to be unsupported for pmtiles.

It was always a bad idea even for mbtiles.

Co-authored-by: Brandon Liu <bdon@bdon.org>
This commit is contained in:
Erica Fischer
2022-12-29 12:00:13 -08:00
committed by GitHub
co-authored by Brandon Liu
parent a374263466
commit 3b2599f587
197 changed files with 2517 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
decode.cpp
+78 -26
View File
@@ -15,7 +15,6 @@
#include <sys/stat.h>
#include <sys/mman.h>
#include <protozero/pbf_reader.hpp>
#include <sys/stat.h>
#include "mvt.hpp"
#include "projection.hpp"
#include "geometry.hpp"
@@ -23,6 +22,7 @@
#include "jsonpull/jsonpull.h"
#include "mbtiles.hpp"
#include "dirtiles.hpp"
#include "pmtiles_file.hpp"
#include "errors.hpp"
int minzoom = 0;
@@ -245,7 +245,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
if (st.st_size < 50 * 1024 * 1024) {
char *map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (map != NULL && map != MAP_FAILED) {
if (strcmp(map, "SQLite format 3") != 0) {
if (strcmp(map, "SQLite format 3") != 0 && strncmp(map, "PMTiles", 7) != 0) {
if (z >= 0) {
std::string s = std::string(map, st.st_size);
handle(s, z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
@@ -272,11 +272,30 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
struct stat st;
std::vector<zxy> tiles;
char *pmtiles_map;
std::vector<pmtiles::entry_zxy> entries;
bool is_pmtiles = false;
if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) {
isdir = true;
db = dirmeta2tmp(fname);
tiles = enumerate_dirtiles(fname, minzoom, maxzoom);
} else if (pmtiles_has_suffix(fname)) {
int pmtiles_fd = open(fname, O_RDONLY | O_CLOEXEC);
pmtiles_map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, pmtiles_fd, 0);
if (pmtiles_map == MAP_FAILED) {
perror("mmap in decode");
exit(EXIT_MEMORY);
}
if (close(pmtiles_fd) != 0) {
perror("close");
exit(EXIT_CLOSE);
}
db = pmtilesmeta2tmp(fname, pmtiles_map);
entries = pmtiles_entries_tms(pmtiles_map, minzoom, maxzoom);
is_pmtiles = true;
} else {
if (sqlite3_open(fname, &db) != SQLITE_OK) {
fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db));
@@ -382,6 +401,26 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, state, coordinate_mode);
}
} else if (is_pmtiles) {
within = 0;
for (auto const &entry : entries) {
if (!pipeline && !stats) {
if (within) {
state.json_comma_newline();
}
within = 1;
}
if (stats) {
if (within) {
state.json_comma_newline();
}
within = 1;
}
std::string s{pmtiles_map + entry.offset, entry.length};
handle(s, entry.z, entry.x, entry.y, to_decode, pipeline, stats, state, coordinate_mode);
}
} else {
const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;";
sqlite3_stmt *stmt;
@@ -447,36 +486,49 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
} else {
int handled = 0;
while (z >= 0 && !handled) {
const char *sql = "SELECT tile_data from tiles where zoom_level = ? and tile_column = ? and tile_row = ?;";
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
exit(EXIT_SQLITE);
}
sqlite3_bind_int(stmt, 1, z);
sqlite3_bind_int(stmt, 2, x);
sqlite3_bind_int(stmt, 3, (1LL << z) - 1 - y);
while (sqlite3_step(stmt) == SQLITE_ROW) {
int len = sqlite3_column_bytes(stmt, 0);
const char *s = (const char *) sqlite3_column_blob(stmt, 0);
if (s == NULL) {
fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n");
if (is_pmtiles) {
uint64_t tile_offset;
uint32_t tile_length;
std::tie(tile_offset, tile_length) = pmtiles_get_tile(pmtiles_map, z, x, y);
if (tile_length > 0) {
if (z != oz) {
fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy);
}
std::string s{pmtiles_map + tile_offset, tile_length};
handle(s, z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
handled = 1;
}
} else {
const char *sql = "SELECT tile_data from tiles where zoom_level = ? and tile_column = ? and tile_row = ?;";
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
exit(EXIT_SQLITE);
}
if (z != oz) {
fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy);
sqlite3_bind_int(stmt, 1, z);
sqlite3_bind_int(stmt, 2, x);
sqlite3_bind_int(stmt, 3, (1LL << z) - 1 - y);
while (sqlite3_step(stmt) == SQLITE_ROW) {
int len = sqlite3_column_bytes(stmt, 0);
const char *s = (const char *) sqlite3_column_blob(stmt, 0);
if (s == NULL) {
fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n");
exit(EXIT_SQLITE);
}
if (z != oz) {
fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy);
}
handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
handled = 1;
}
handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
handled = 1;
sqlite3_finalize(stmt);
}
sqlite3_finalize(stmt);
z--;
x /= 2;
y /= 2;