diff --git a/.gitignore b/.gitignore index 4581ef2e..4bf996af 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ *.exe *.out *.app + +# Mac +.DS_Store \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1d3f9f..3d8bcc80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.16.15 + +* Add --output-to-directory and --no-tile-compression options + ## 1.16.14 * Add --description option for mbtiles metadata diff --git a/Makefile b/Makefile index 8e3f7fa9..f9d032bd 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o +tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o rawtiles.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o @@ -78,7 +78,7 @@ indent: TESTS = $(wildcard tests/*/out/*.json) SPACE = $(NULL) $(NULL) -test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) parallel-test pbf-test join-test enumerate-test decode-test unit +test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test unit ./unit # Work around Makefile and filename punctuation limits: _ for space, @ for :, % for / @@ -111,6 +111,11 @@ parallel-test: cmp tests/parallel/linear-file.json tests/parallel/parallel-pipes.json rm tests/parallel/*.mbtiles tests/parallel/*.json +raw-tiles-test: + ./tippecanoe -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC + diff -x '.*' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare + rm -rf tests/raw-tiles/raw-tiles + decode-test: mkdir -p tests/muni/decode ./tippecanoe -z11 -Z11 -f -o tests/muni/decode/multi.mbtiles tests/muni/*.json diff --git a/README.md b/README.md index e9c7f79b..d9b80788 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Options ### File control * -o _file_.mbtiles or --output=_file_.mbtiles: Name the output file. + * -e _directory_ or --output-directory=_directory_: Write tiles to the specified *directory* instead of to an mbtiles file. * -f or --force: Delete the mbtiles file if it already exists instead of giving an error * -F or --allow-existing: Proceed (without deleting existing data) if the metadata or tiles table already exists or if metadata fields can't be set @@ -147,6 +148,7 @@ resolution is obtained than by using a smaller _maxzoom_ or _detail_. * -pc or --no-clipping: Don't clip features to the size of the tile. If a feature overlaps the tile's bounds or buffer at all, it is included completely. Be careful: this can produce very large tilesets, especially with large polygons. * -pD or --no-duplication: As with --no-clipping, each feature is included intact instead of cut to tile boundaries. In addition, it is included only in a single tile per zoom level rather than potentially in multiple copies. Clients of the tileset must check adjacent tiles (possibly some distance away) to ensure they have all features. * -pt or --no-tiny-polygon-reduction: Don't combine the area of very small polygons into small squares that represent their combined area. + * -pC or --no-tile-compression: Don't compress the PBF vector tile data. * -q or --quiet: Work quietly instead of reporting progress Environment diff --git a/main.cpp b/main.cpp index 418ea7f5..78c61281 100644 --- a/main.cpp +++ b/main.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -1005,7 +1006,7 @@ void choose_first_zoom(long long *file_bbox, struct reader *reader, unsigned *iz } } -int read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set *exclude, std::set *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description) { +int read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description) { int ret = EXIT_SUCCESS; struct reader reader[CPUS]; @@ -1779,7 +1780,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo } unsigned midx = 0, midy = 0; - int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps); + int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, outdir, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); @@ -1843,7 +1844,8 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo ai->second.minzoom = minzoom; ai->second.maxzoom = maxzoom; } - mbtiles_write_metadata(outdb, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); + + mbtiles_write_metadata(outdb, outdir, fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, forcetable, attribution, merged_lm, true, description); return ret; } @@ -1872,7 +1874,9 @@ int main(int argc, char **argv) { char *name = NULL; char *description = NULL; char *layername = NULL; - char *outdir = NULL; + char *out_mbtiles = NULL; + char *out_directory = NULL; + sqlite3 *outdb = NULL; int maxzoom = 14; int minzoom = 0; int basezoom = -1; @@ -1898,6 +1902,7 @@ int main(int argc, char **argv) { static struct option long_options[] = { {"output", required_argument, 0, 'o'}, + {"output-to-directory", required_argument, 0, 'e'}, {"name", required_argument, 0, 'n'}, {"description", required_argument, 0, 'N'}, @@ -1955,6 +1960,7 @@ int main(int argc, char **argv) { {"no-clipping", no_argument, &prevent[P_CLIPPING], 1}, {"no-duplication", no_argument, &prevent[P_DUPLICATION], 1}, {"no-tiny-polygon-reduction", no_argument, &prevent[P_TINY_POLYGON_REDUCTION], 1}, + {"no-tile-compression", no_argument, &prevent[P_TILE_COMPRESSION], 1}, {0, 0, 0, 0}, }; @@ -1977,7 +1983,7 @@ int main(int argc, char **argv) { } } - while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:", long_options, NULL)) != -1) { + while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:e:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:", long_options, NULL)) != -1) { switch (i) { case 0: break; @@ -2054,7 +2060,11 @@ int main(int argc, char **argv) { break; case 'o': - outdir = optarg; + out_mbtiles = optarg; + break; + + case 'e': + out_directory = optarg; break; case 'x': @@ -2238,16 +2248,24 @@ int main(int argc, char **argv) { fprintf(stderr, "Forcing -g0 since -B or -r is not known\n"); } - if (outdir == NULL) { - fprintf(stderr, "%s: must specify -o out.mbtiles\n", argv[0]); + if (out_mbtiles == NULL && out_directory == NULL) { + fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory\n", argv[0]); exit(EXIT_FAILURE); } - if (force) { - unlink(outdir); + if (out_mbtiles != NULL && out_directory != NULL) { + fprintf(stderr, "%s: Options -o and -e cannot be used together\n", argv[0]); + exit(EXIT_FAILURE); + } + + if (out_mbtiles != NULL) { + if (force) { + unlink(out_mbtiles); + } + + outdb = mbtiles_open(out_mbtiles, argv, forcetable); } - sqlite3 *outdb = mbtiles_open(outdir, argv, forcetable); int ret = EXIT_SUCCESS; for (i = optind; i < argc; i++) { @@ -2272,9 +2290,11 @@ int main(int argc, char **argv) { long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0}; - ret = read_input(sources, name ? name : outdir, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description); + ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_directory, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_directory, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description); - mbtiles_close(outdb, argv); + if (outdb != NULL) { + mbtiles_close(outdb, argv); + } #ifdef MTRACE muntrace(); diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 5d369249..37a44683 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -60,12 +60,16 @@ specified, the files are all merged into the single named layer, even if they tr \-n \fIname\fP or \-\-name=\fIname\fP: Human\-readable name for the tileset (default file.json) .IP \(bu 2 \-A \fItext\fP or \-\-attribution=\fItext\fP: Attribution (HTML) to be shown with maps that use data from this tileset. +.IP \(bu 2 +\-N \fIdescription\fP or \-\-description=\fIdescription\fP: Description for the tileset (default file.mbtiles) .RE .SS File control .RS .IP \(bu 2 \-o \fIfile\fP\&.mbtiles or \-\-output=\fIfile\fP\&.mbtiles: Name the output file. .IP \(bu 2 +\-e \fIdirectory\fP or \-\-output\-directory=\fIdirectory\fP: Write tiles to the specified \fIdirectory\fP instead of to an mbtiles file. +.IP \(bu 2 \-f or \-\-force: Delete the mbtiles file if it already exists instead of giving an error .IP \(bu 2 \-F or \-\-allow\-existing: Proceed (without deleting existing data) if the metadata or tiles table already exists @@ -162,6 +166,8 @@ which may not be what you want. \-an or \-\-drop\-smallest\-as\-needed: Dynamically drop the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level to keep large tiles under the 500K size limit. This option will not work for point features. .IP \(bu 2 \-aL or \-\-grid\-low\-zooms: At all zoom levels below \fImaxzoom\fP, snap all lines and polygons to a stairstep grid instead of allowing diagonals. You will also want to specify a tile resolution, probably \fB\fC\-D8\fR\&. This option provides a way to display continuous parcel, gridded, or binned data at low zooms without overwhelming the tiles with tiny polygons, since features will either get stretched out to the grid unit or lost entirely, depending on how they happened to be aligned in the original data. +.IP \(bu 2 +\-aw or \-\-detect\-longitude\-wraparound: Detect when adjacent points within a feature jump to the other side of the world, and try to fix the geometry. .RE .SS Doing less .RS @@ -178,7 +184,7 @@ which may not be what you want. .IP \(bu 2 \-pi or \-\-preserve\-input\-order: Preserve the original input order of features as the drawing order instead of ordering geographically. (This is implemented as a restoration of the original order at the end, so that dot\-dropping is still geographic, which means it also undoes \-ao). .IP \(bu 2 -\-pp or \-\-no\-polygon\-splitting: Don't split complex polygons (over 700 vertices after simplification) into multiple features. +\-pp or \-\-no\-polygon\-splitting: This no longer has any effect. .IP \(bu 2 \-pc or \-\-no\-clipping: Don't clip features to the size of the tile. If a feature overlaps the tile's bounds or buffer at all, it is included completely. Be careful: this can produce very large tilesets, especially with large polygons. .IP \(bu 2 @@ -186,6 +192,8 @@ which may not be what you want. .IP \(bu 2 \-pt or \-\-no\-tiny\-polygon\-reduction: Don't combine the area of very small polygons into small squares that represent their combined area. .IP \(bu 2 +\-pC or \-\-no\-tile\-compression: Don't compress the PBF vector tile data. +.IP \(bu 2 \-q or \-\-quiet: Work quietly instead of reporting progress .RE .SH Environment @@ -301,9 +309,6 @@ have their probability diffused, so that some of them will be drawn as a square this minimum size and others will not be drawn at all, preserving the total area that all of them should have had together. .PP -Any polygons that have over 700 vertices after line simplification will be split into -multiple features so they can be rendered efficiently, unless you use \-pp to prevent this. -.PP Features in the same tile that share the same type and attributes are coalesced together into a single geometry if you use \fB\fC\-\-coalesce\fR\&. You are strongly encouraged to use \-x to exclude any unnecessary properties to reduce wasted file size. diff --git a/mbtiles.cpp b/mbtiles.cpp index 0a364da7..ab9aeb31 100644 --- a/mbtiles.cpp +++ b/mbtiles.cpp @@ -131,11 +131,23 @@ bool type_and_string::operator<(const type_and_string &o) const { return false; } -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description) { +void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description) { char *sql, *err; + sqlite3 *db = outdb; + if (outdb == NULL) { + if (sqlite3_open("", &db) != SQLITE_OK) { + fprintf(stderr, "Temporary db: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + if (sqlite3_exec(db, "CREATE TABLE metadata (name text, value text);", NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "Create metadata table: %s\n", err); + exit(EXIT_FAILURE); + } + } + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('name', %Q);", fname); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set name in metadata: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -144,7 +156,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('description', %Q);", description != NULL ? description : fname); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set description in metadata: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -153,7 +165,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('version', %d);", 2); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set version : %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -162,7 +174,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('minzoom', %d);", minzoom); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set minzoom: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -171,7 +183,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);", maxzoom); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set maxzoom: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -180,7 +192,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');", midlon, midlat, maxzoom); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set center: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -189,7 +201,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');", minlon, minlat, maxlon, maxlat); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set bounds: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -198,7 +210,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int sqlite3_free(sql); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", "overlay"); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set type: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -208,7 +220,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int if (attribution != NULL) { sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('attribution', %Q);", attribution); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set type: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -218,7 +230,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int } sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('format', %Q);", vector ? "pbf" : "png"); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set format: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -276,7 +288,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int aprintf(&buf, " ] }"); sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('json', %Q);", buf.c_str()); - if (sqlite3_exec(outdb, sql, NULL, NULL, &err) != SQLITE_OK) { + if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { fprintf(stderr, "set json: %s\n", err); if (!forcetable) { exit(EXIT_FAILURE); @@ -284,6 +296,45 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int } sqlite3_free(sql); } + + if (outdir != NULL) { + std::string metadata = std::string(outdir) + "/metadata.json"; + FILE *fp = fopen(metadata.c_str(), "w"); + if (fp == NULL) { + perror(metadata.c_str()); + exit(EXIT_FAILURE); + } + + fprintf(fp, "{\n"); + + sqlite3_stmt *stmt; + bool first = true; + if (sqlite3_prepare_v2(db, "SELECT name, value from metadata;", -1, &stmt, NULL) == SQLITE_OK) { + while (sqlite3_step(stmt) == SQLITE_ROW) { + std::string key, value; + + quote(&key, (const char *) sqlite3_column_text(stmt, 0)); + quote(&value, (const char *) sqlite3_column_text(stmt, 1)); + + if (!first) { + fprintf(fp, ",\n"); + } + fprintf(fp, " \"%s\": \"%s\"", key.c_str(), value.c_str()); + first = false; + } + sqlite3_finalize(stmt); + } + + fprintf(fp, "\n}\n"); + fclose(fp); + } + + if (outdb == NULL) { + if (sqlite3_close(db) != SQLITE_OK) { + fprintf(stderr, "Could not close temp database: %s\n", sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + } } void mbtiles_close(sqlite3 *outdb, char **argv) { diff --git a/mbtiles.hpp b/mbtiles.hpp index 4fb18b18..dbe93cd1 100644 --- a/mbtiles.hpp +++ b/mbtiles.hpp @@ -20,7 +20,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable); void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size); -void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description); +void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map const &layermap, bool vector, const char *description); void mbtiles_close(sqlite3 *outdb, char **argv); diff --git a/mvt.cpp b/mvt.cpp index f11164ba..22198e9f 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -367,10 +367,7 @@ std::string mvt_tile::encode() { writer.add_message(3, layer_string); } - std::string compressed; - compress(data, compressed); - - return compressed; + return data; } bool mvt_value::operator<(const mvt_value &o) const { diff --git a/options.hpp b/options.hpp index 4079369b..6081d7dc 100644 --- a/options.hpp +++ b/options.hpp @@ -25,6 +25,7 @@ #define P_CLIPPING ((int) 'c') #define P_DUPLICATION ((int) 'D') #define P_TINY_POLYGON_REDUCTION ((int) 't') +#define P_TILE_COMPRESSION ((int) 'C') extern int prevent[256]; extern int additional[256]; diff --git a/rawtiles.cpp b/rawtiles.cpp new file mode 100644 index 00000000..1c7ecc4a --- /dev/null +++ b/rawtiles.cpp @@ -0,0 +1,20 @@ +#include +#include +#include +#include +#include "rawtiles.hpp" + +void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) { + mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO); + std::string curdir(outdir); + std::string slash("/"); + std::string newdir = curdir + slash + std::to_string(z); + mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(tx); + mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO); + newdir = newdir + "/" + std::to_string(ty) + ".pbf"; + + std::ofstream pbfFile(newdir, std::ios::out | std::ios::binary); + pbfFile.write(pbf.data(), pbf.size()); + pbfFile.close(); +} diff --git a/rawtiles.hpp b/rawtiles.hpp new file mode 100644 index 00000000..1a332c94 --- /dev/null +++ b/rawtiles.hpp @@ -0,0 +1 @@ +void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf); diff --git a/tests/raw-tiles/compare/0/0/0.pbf b/tests/raw-tiles/compare/0/0/0.pbf new file mode 100644 index 00000000..3f518d33 Binary files /dev/null and b/tests/raw-tiles/compare/0/0/0.pbf differ diff --git a/tests/raw-tiles/compare/1/0/0.pbf b/tests/raw-tiles/compare/1/0/0.pbf new file mode 100644 index 00000000..49b703d2 Binary files /dev/null and b/tests/raw-tiles/compare/1/0/0.pbf differ diff --git a/tests/raw-tiles/compare/10/163/366.pbf b/tests/raw-tiles/compare/10/163/366.pbf new file mode 100644 index 00000000..9fd8fe08 Binary files /dev/null and b/tests/raw-tiles/compare/10/163/366.pbf differ diff --git a/tests/raw-tiles/compare/11/326/732.pbf b/tests/raw-tiles/compare/11/326/732.pbf new file mode 100644 index 00000000..ae7fcdba Binary files /dev/null and b/tests/raw-tiles/compare/11/326/732.pbf differ diff --git a/tests/raw-tiles/compare/12/652/1464.pbf b/tests/raw-tiles/compare/12/652/1464.pbf new file mode 100644 index 00000000..173f8208 Binary files /dev/null and b/tests/raw-tiles/compare/12/652/1464.pbf differ diff --git a/tests/raw-tiles/compare/12/652/1465.pbf b/tests/raw-tiles/compare/12/652/1465.pbf new file mode 100644 index 00000000..385f3357 Binary files /dev/null and b/tests/raw-tiles/compare/12/652/1465.pbf differ diff --git a/tests/raw-tiles/compare/13/1304/2928.pbf b/tests/raw-tiles/compare/13/1304/2928.pbf new file mode 100644 index 00000000..22531b0a Binary files /dev/null and b/tests/raw-tiles/compare/13/1304/2928.pbf differ diff --git a/tests/raw-tiles/compare/13/1304/2929.pbf b/tests/raw-tiles/compare/13/1304/2929.pbf new file mode 100644 index 00000000..6aad49f6 Binary files /dev/null and b/tests/raw-tiles/compare/13/1304/2929.pbf differ diff --git a/tests/raw-tiles/compare/13/1304/2930.pbf b/tests/raw-tiles/compare/13/1304/2930.pbf new file mode 100644 index 00000000..cecb84fe Binary files /dev/null and b/tests/raw-tiles/compare/13/1304/2930.pbf differ diff --git a/tests/raw-tiles/compare/14/2608/5856.pbf b/tests/raw-tiles/compare/14/2608/5856.pbf new file mode 100644 index 00000000..b279fba5 Binary files /dev/null and b/tests/raw-tiles/compare/14/2608/5856.pbf differ diff --git a/tests/raw-tiles/compare/14/2608/5857.pbf b/tests/raw-tiles/compare/14/2608/5857.pbf new file mode 100644 index 00000000..2272a830 Binary files /dev/null and b/tests/raw-tiles/compare/14/2608/5857.pbf differ diff --git a/tests/raw-tiles/compare/14/2609/5860.pbf b/tests/raw-tiles/compare/14/2609/5860.pbf new file mode 100644 index 00000000..d8f0684d Binary files /dev/null and b/tests/raw-tiles/compare/14/2609/5860.pbf differ diff --git a/tests/raw-tiles/compare/2/0/1.pbf b/tests/raw-tiles/compare/2/0/1.pbf new file mode 100644 index 00000000..ecd69b4e Binary files /dev/null and b/tests/raw-tiles/compare/2/0/1.pbf differ diff --git a/tests/raw-tiles/compare/3/1/2.pbf b/tests/raw-tiles/compare/3/1/2.pbf new file mode 100644 index 00000000..acf10f52 Binary files /dev/null and b/tests/raw-tiles/compare/3/1/2.pbf differ diff --git a/tests/raw-tiles/compare/4/2/5.pbf b/tests/raw-tiles/compare/4/2/5.pbf new file mode 100644 index 00000000..2e4698bd Binary files /dev/null and b/tests/raw-tiles/compare/4/2/5.pbf differ diff --git a/tests/raw-tiles/compare/5/5/11.pbf b/tests/raw-tiles/compare/5/5/11.pbf new file mode 100644 index 00000000..40ab997a Binary files /dev/null and b/tests/raw-tiles/compare/5/5/11.pbf differ diff --git a/tests/raw-tiles/compare/6/10/22.pbf b/tests/raw-tiles/compare/6/10/22.pbf new file mode 100644 index 00000000..75c556bd Binary files /dev/null and b/tests/raw-tiles/compare/6/10/22.pbf differ diff --git a/tests/raw-tiles/compare/7/20/45.pbf b/tests/raw-tiles/compare/7/20/45.pbf new file mode 100644 index 00000000..b54f433c Binary files /dev/null and b/tests/raw-tiles/compare/7/20/45.pbf differ diff --git a/tests/raw-tiles/compare/8/40/91.pbf b/tests/raw-tiles/compare/8/40/91.pbf new file mode 100644 index 00000000..26c2f2aa Binary files /dev/null and b/tests/raw-tiles/compare/8/40/91.pbf differ diff --git a/tests/raw-tiles/compare/9/81/183.pbf b/tests/raw-tiles/compare/9/81/183.pbf new file mode 100644 index 00000000..dd0721fd Binary files /dev/null and b/tests/raw-tiles/compare/9/81/183.pbf differ diff --git a/tests/raw-tiles/compare/metadata.json b/tests/raw-tiles/compare/metadata.json new file mode 100644 index 00000000..c83c17c6 --- /dev/null +++ b/tests/raw-tiles/compare/metadata.json @@ -0,0 +1,12 @@ +{ + "name": "tests/raw-tiles/raw-tiles", + "description": "tests/raw-tiles/raw-tiles", + "version": "2", + "minzoom": "0", + "maxzoom": "14", + "center": "-122.662354,45.514045,14", + "bounds": "-122.682427,45.512332,-122.654961,45.569975", + "type": "overlay", + "format": "pbf", + "json": "{\"vector_layers\": [ { \"id\": \"hackspotsgeojson\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 14, \"fields\": {\"Address\": \"String\", \"Name\": \"String\", \"Notes\": \"String\"} } ] }" +} diff --git a/tests/raw-tiles/hackspots.geojson b/tests/raw-tiles/hackspots.geojson new file mode 100644 index 00000000..1f6e9818 --- /dev/null +++ b/tests/raw-tiles/hackspots.geojson @@ -0,0 +1,65 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "Name": "Albina Press", + "Address": "4637 N Albina Ave Portland, OR 97217", + "Notes": "usually busy, outlets on side wall only" + }, + "geometry": { + "type": "Point", + "coordinates": [ + -122.67516911029816, + 45.55673233031101 + ] + } + }, + { + "type": "Feature", + "properties": { + "Name": "Arbor Lodge", + "Address": "1507 N Rosa Parks Way Portland, OR 97217", + "Notes": "" + }, + "geometry": { + "type": "Point", + "coordinates": [ + -122.68242716789246, + 45.56997505986905 + ] + } + }, + { + "type": "Feature", + "properties": { + "Name": "Three Friends Coffeehouse", + "Address": "201 SE 12th Ave, Portland, OR 97214", + "Notes": "" + }, + "geometry": { + "type": "Point", + "coordinates": [ + -122.65496134757996, + 45.5212590460849 + ] + } + }, + { + "type": "Feature", + "properties": { + "Name": "Lucky Labrador Brew Pub", + "Address": "915 SE Hawthorne Blvd. Portland, OR 97214", + "Notes": "Dog friendly" + }, + "geometry": { + "type": "Point", + "coordinates": [ + -122.65625953674315, + 45.512331780814186 + ] + } + } + ] +} \ No newline at end of file diff --git a/tile-join.cpp b/tile-join.cpp index 4920f2da..6ca1617c 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -750,7 +750,7 @@ int main(int argc, char **argv) { decode(readers, csv, layermap, outdb, &st, header, mapping, exclude, ifmatched, attribution, description); - mbtiles_write_metadata(outdb, outfile, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str()); + mbtiles_write_metadata(outdb, NULL, outfile, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str()); mbtiles_close(outdb, argv); return 0; diff --git a/tile.cpp b/tile.cpp index f9723276..9d43bc50 100644 --- a/tile.cpp +++ b/tile.cpp @@ -22,6 +22,7 @@ #include #include "mvt.hpp" #include "mbtiles.hpp" +#include "rawtiles.hpp" #include "geometry.hpp" #include "tile.hpp" #include "pool.hpp" @@ -1184,6 +1185,7 @@ struct write_tile_args { int min_detail; int basezoom; sqlite3 *outdb; + const char *outdir; double droprate; int buffer; const char *fname; @@ -1221,7 +1223,7 @@ struct write_tile_args { double fraction_out; }; -long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, write_tile_args *arg) { +long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, write_tile_args *arg) { int line_detail; double merge_fraction = 1; double mingap_fraction = 1; @@ -1837,7 +1839,14 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s } } - std::string compressed = tile.encode(); + std::string compressed; + std::string pbf = tile.encode(); + + if (!prevent[P_TILE_COMPRESSION]) { + compress(pbf, compressed); + } else { + compressed = pbf; + } if (compressed.size() > max_tile_size && !prevent[P_KILOBYTE_LIMIT]) { if (!quiet) { @@ -1909,7 +1918,11 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s exit(EXIT_FAILURE); } - mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); + if (outdb != NULL) { + mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); + } else if (outdir != NULL) { + write_raw_tile(outdir, z, tx, ty, compressed); + } if (pthread_mutex_unlock(&db_lock) != 0) { perror("pthread_mutex_unlock"); @@ -1971,7 +1984,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg); + long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->outdir, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg); if (len < 0) { int *err = &arg->err; @@ -2036,7 +2049,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps) { // Table to map segment and layer number back to layer name std::vector> layer_unmaps; for (size_t seg = 0; seg < layermaps.size(); seg++) { @@ -2171,6 +2184,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].min_detail = min_detail; args[thread].basezoom = basezoom; args[thread].outdb = outdb; // locked with db_lock + args[thread].outdir = outdir; args[thread].droprate = droprate; args[thread].buffer = buffer; args[thread].fname = fname; diff --git a/tile.hpp b/tile.hpp index ffbb9607..2ae8d7f4 100644 --- a/tile.hpp +++ b/tile.hpp @@ -1,5 +1,5 @@ -long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers); +long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector > &layermap); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/version.hpp b/version.hpp index 3b4aad79..b44031b4 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.16.13\n" +#define VERSION "tippecanoe v1.16.14\n"