From 7372a2c4bc3f49d9a6a569fa2e47244f84c34a78 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 15 Dec 2015 11:56:49 -0800 Subject: [PATCH 01/14] Add basezoom parameter for dot-dropping, independent of maxzoom --- README.md | 9 ++++++--- geojson.c | 33 +++++++++++++++++++++------------ tile.cc | 8 ++------ tile.h | 2 +- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1978d781..fa17f00d 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,9 @@ Options ### Zoom levels and resolution - * -z _zoom_: Base (maxzoom) zoom level (default 14) + * -z _zoom_: Max zoom level (default 14) * -Z _zoom_: Lowest (minzoom) zoom level (default 0) + * -B _zoom_: Base zoom, the level at which point data is complete (default maxzoom) * -d _detail_: Detail at base zoom level (default 12, for tile resolution of 4096) * -D _detail_: Detail at lower zoom levels (default 12, for tile resolution of 4096) * -m _detail_: Minimum detail that it will try if tiles are too big at regular detail (default 7) @@ -142,7 +143,7 @@ coordinated with the base zoom level and dot-dropping rate. You can use this she calculate the appropriate marker-width at high zoom levels to match the fraction of dots that were dropped at low zoom levels. -If you used `-z` to change the base zoom level or `-r` to change the +If you used `-B` or `-z` to change the base zoom level or `-r` to change the dot-dropping rate, replace them in the `basezoom` and `rate` below. awk 'BEGIN { @@ -167,7 +168,9 @@ Geometric simplifications At every zoom level, line and polygon features are subjected to Douglas-Peucker simplification to the resolution of the tile. -For point features, it drops 1/2.5 of the dots for each zoom level above the base. +For point features, it drops 1/2.5 of the dots for each zoom level above the +point base zoom (which is normally the same as the `-z` max zoom, but can be +a different zoom specified with `-B` if you have precise but sparse data). I don't know why 2.5 is the appropriate number, but the densities of many different data sets fall off at about this same rate. You can use -r to specify a different rate. diff --git a/geojson.c b/geojson.c index c933657e..aaadd510 100644 --- a/geojson.c +++ b/geojson.c @@ -437,7 +437,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, char *s, c return off; } -int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, json_pull *jp, long long *seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int maxzoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe) { +int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, json_pull *jp, long long *seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe) { json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == NULL) { static int warned = 0; @@ -587,7 +587,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha if (r == 0) { r = .00000001; } - minzoom = maxzoom - floor(log(r) / -log(droprate)); + minzoom = basezoom - floor(log(r) / -log(droprate)); } serialize_byte(geomfile, minzoom, geompos, fname); @@ -626,7 +626,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha return 1; } -void parse_json(json_pull *jp, const char *reading, long long *seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int maxzoom, int layer, double droprate, long long *file_bbox) { +void parse_json(json_pull *jp, const char *reading, long long *seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; @@ -691,7 +691,7 @@ void parse_json(json_pull *jp, const char *reading, long long *seq, long long *m } found_geometries++; - serialize_geometry(j, NULL, reading, jp, seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, layer, droprate, file_bbox, NULL); + serialize_geometry(j, NULL, reading, jp, seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, basezoom, layer, droprate, file_bbox, NULL); json_free(j); continue; } @@ -726,10 +726,10 @@ void parse_json(json_pull *jp, const char *reading, long long *seq, long long *m if (geometries != NULL) { int g; for (g = 0; g < geometries->length; g++) { - serialize_geometry(geometries->array[g], properties, reading, jp, seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, layer, droprate, file_bbox, tippecanoe); + serialize_geometry(geometries->array[g], properties, reading, jp, seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, basezoom, layer, droprate, file_bbox, tippecanoe); } } else { - serialize_geometry(geometry, properties, reading, jp, seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, layer, droprate, file_bbox, tippecanoe); + serialize_geometry(geometry, properties, reading, jp, seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, basezoom, layer, droprate, file_bbox, tippecanoe); } json_free(j); @@ -738,7 +738,7 @@ void parse_json(json_pull *jp, const char *reading, long long *seq, long long *m } } -int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional) { +int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional) { int ret = EXIT_SUCCESS; char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; @@ -866,7 +866,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max layer = source; } - parse_json(jp, reading, &seq, &metapos, &geompos, &indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, layer, droprate, file_bbox); + parse_json(jp, reading, &seq, &metapos, &geompos, &indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, basezoom, layer, droprate, file_bbox); json_end(jp); fclose(fp); @@ -1138,7 +1138,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", seq, (long long) geomst.st_size, (long long) metast.st_size, poolfile->off); } - int written = traverse_zooms(fd, size, meta, stringpool, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail); + int written = traverse_zooms(fd, size, meta, stringpool, file_keys, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); @@ -1224,6 +1224,7 @@ int main(int argc, char **argv) { char *outdir = NULL; int maxzoom = 14; int minzoom = 0; + int basezoom = -1; int force = 0; double droprate = 2.5; double gamma = 0; @@ -1242,7 +1243,7 @@ int main(int argc, char **argv) { additional[i] = 0; } - while ((i = getopt(argc, argv, "l:n:z:Z:d:D:m:o:x:y:r:b:fXt:g:p:vqa:")) != -1) { + while ((i = getopt(argc, argv, "l:n:z:Z:d:D:m:o:x:y:r:b:fXt:g:p:vqa:B:")) != -1) { switch (i) { case 'n': name = optarg; @@ -1260,6 +1261,10 @@ int main(int argc, char **argv) { minzoom = atoi(optarg); break; + case 'B': + basezoom = atoi(optarg); + break; + case 'd': full_detail = atoi(optarg); break; @@ -1332,7 +1337,7 @@ int main(int argc, char **argv) { exit(EXIT_FAILURE); default: - fprintf(stderr, "Usage: %s -o out.mbtiles [-n name] [-l layername] [-z maxzoom] [-Z minzoom] [-d detail] [-D lower-detail] [-m min-detail] [-x excluded-field ...] [-y included-field ...] [-X] [-r droprate] [-b buffer] [-t tmpdir] [-a rco] [-p sfkld] [-q] [file.json ...]\n", argv[0]); + fprintf(stderr, "Usage: %s -o out.mbtiles [-n name] [-l layername] [-z maxzoom] [-Z minzoom] [-B basezoom] [-d detail] [-D lower-detail] [-m min-detail] [-x excluded-field ...] [-y included-field ...] [-X] [-r droprate] [-b buffer] [-t tmpdir] [-a rco] [-p sfkld] [-q] [file.json ...]\n", argv[0]); exit(EXIT_FAILURE); } } @@ -1342,6 +1347,10 @@ int main(int argc, char **argv) { exit(EXIT_FAILURE); } + if (basezoom < 0) { + basezoom = maxzoom; + } + if (full_detail <= 0) { full_detail = 12; } @@ -1365,7 +1374,7 @@ int main(int argc, char **argv) { sqlite3 *outdb = mbtiles_open(outdir, argv); int ret = EXIT_SUCCESS; - ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional); + ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, basezoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional); mbtiles_close(outdb, argv); diff --git a/tile.cc b/tile.cc index 8784a588..38c37904 100644 --- a/tile.cc +++ b/tile.cc @@ -906,8 +906,6 @@ struct write_tile_args { int buffer; const char *fname; FILE **geomfile; - int file_minzoom; - int file_maxzoom; double todo; volatile long long *along; double gamma; @@ -999,7 +997,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail) { int i; for (i = 0; i <= maxzoom; i++) { long long most = 0; @@ -1098,7 +1096,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].metabase = metabase; args[thread].stringpool = stringpool; args[thread].min_detail = min_detail; - args[thread].basezoom = maxzoom; // XXX rename? + args[thread].basezoom = basezoom; args[thread].file_keys = file_keys; // locked with var_lock args[thread].layernames = layernames; args[thread].outdb = outdb; // locked with db_lock @@ -1106,8 +1104,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].buffer = buffer; args[thread].fname = fname; args[thread].geomfile = sub + thread * (TEMP_FILES / threads); - args[thread].file_minzoom = minzoom; - args[thread].file_maxzoom = maxzoom; args[thread].todo = todo; args[thread].along = &along; // locked with var_lock args[thread].gamma = gamma; diff --git a/tile.h b/tile.h index b2746e76..2ac893fa 100644 --- a/tile.h +++ b/tile.h @@ -27,7 +27,7 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type); long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail); extern unsigned initial_x, initial_y; extern int geometry_scale; From 76de0c82d2b6f5b301495079600b5b4000f2368a Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 15 Dec 2015 12:00:05 -0800 Subject: [PATCH 02/14] Further cleanup of minzoom- and maxzoom-related variable names --- geojson.c | 14 +++++++------- tile.cc | 18 +++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/geojson.c b/geojson.c index aaadd510..a9e43bfb 100644 --- a/geojson.c +++ b/geojson.c @@ -564,19 +564,19 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha serialize_byte(geomfile, VT_END, geompos, fname); /* - * Note that minzoom for lines is the dimension + * Note that feature_minzoom for lines is the dimension * of the geometry in world coordinates, but * for points is the lowest zoom level (in tiles, * not in pixels) at which it should be drawn. * * So a line that is too small for, say, z8 - * will have minzoom of 18 (if tile detail is 10), + * will have feature_minzoom of 18 (if tile detail is 10), * not 8. */ - int minzoom = 0; + int feature_minzoom = 0; if (mb_geometry[t] == VT_LINE) { - for (minzoom = 0; minzoom < 31; minzoom++) { - unsigned mask = 1 << (32 - (minzoom + 1)); + for (feature_minzoom = 0; feature_minzoom < 31; feature_minzoom++) { + unsigned mask = 1 << (32 - (feature_minzoom + 1)); if (((bbox[0] & mask) != (bbox[2] & mask)) || ((bbox[1] & mask) != (bbox[3] & mask))) { break; @@ -587,10 +587,10 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha if (r == 0) { r = .00000001; } - minzoom = basezoom - floor(log(r) / -log(droprate)); + feature_minzoom = basezoom - floor(log(r) / -log(droprate)); } - serialize_byte(geomfile, minzoom, geompos, fname); + serialize_byte(geomfile, feature_minzoom, geompos, fname); struct index index; index.start = geomstart; diff --git a/tile.cc b/tile.cc index 38c37904..25948a84 100644 --- a/tile.cc +++ b/tile.cc @@ -362,8 +362,8 @@ struct sll { } }; -void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom) { - if (geom.size() > 0 && nextzoom <= file_maxzoom) { +void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom) { + if (geom.size() > 0 && nextzoom <= maxzoom) { int xo, yo; int span = 1 << (nextzoom - z); @@ -463,7 +463,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb } } -long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, char *prevent, char *additional, int child_shards) { +long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, char *prevent, char *additional, int child_shards) { int line_detail; double fraction = 1; @@ -481,9 +481,9 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsi } int nextzoom = z + 1; - if (nextzoom < file_minzoom) { - if (z + max_zoom_increment > file_minzoom) { - nextzoom = file_minzoom; + if (nextzoom < minzoom) { + if (z + max_zoom_increment > minzoom) { + nextzoom = minzoom; } else { nextzoom = z + max_zoom_increment; } @@ -566,7 +566,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsi signed char feature_minzoom; deserialize_byte(geoms, &feature_minzoom); - double progress = floor((((*geoms - geomstart + *along) / (double) todo) + z) / (file_maxzoom + 1) * 1000) / 10; + double progress = floor((((*geoms - geomstart + *along) / (double) todo) + z) / (maxzoom + 1) * 1000) / 10; if (progress >= oprogress + 0.1) { if (!quiet) { fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); @@ -630,10 +630,10 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsi } if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */ - rewrite(geom, z, nextzoom, file_maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, child_shards, max_zoom_increment, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom); + rewrite(geom, z, nextzoom, maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, child_shards, max_zoom_increment, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom); } - if (z < file_minzoom) { + if (z < minzoom) { continue; } From fee8b33bc8706eadff39c05a6d68450534f72aae Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 15 Dec 2015 12:08:07 -0800 Subject: [PATCH 03/14] Fix a place where it was still passing around maxzoom as basezoom --- tile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tile.cc b/tile.cc index 25948a84..22f40680 100644 --- a/tile.cc +++ b/tile.cc @@ -961,7 +961,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(&geom, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->maxzoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards); + long long len = write_tile(&geom, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards); if (len < 0) { int *err = (int *) malloc(sizeof(int)); From 8e2c20265f065c2a67da5ebbac50ddd1c9c99253 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 15 Dec 2015 12:17:36 -0800 Subject: [PATCH 04/14] Try to clarify documentation --- README.md | 10 +++++----- man/tippecanoe.1 | 16 ++++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index fa17f00d..04f86e35 100644 --- a/README.md +++ b/README.md @@ -66,10 +66,10 @@ Options ### Zoom levels and resolution - * -z _zoom_: Max zoom level (default 14) - * -Z _zoom_: Lowest (minzoom) zoom level (default 0) - * -B _zoom_: Base zoom, the level at which point data is complete (default maxzoom) - * -d _detail_: Detail at base zoom level (default 12, for tile resolution of 4096) + * -z _zoom_: Maxzoom: the highest zoom level for which tiles are generated (default 14) + * -Z _zoom_: Minzoom: the lowest zoom level for which tiles are generated (default 0) + * -B _zoom_: Base zoom, the level at and above which all points are included in the tiles (default maxzoom) + * -d _detail_: Detail at max zoom level (default 12, for tile resolution of 4096) * -D _detail_: Detail at lower zoom levels (default 12, for tile resolution of 4096) * -m _detail_: Minimum detail that it will try if tiles are too big at regular detail (default 7) * -b _pixels_: Buffer size where features are duplicated from adjacent tiles. Units are "screen pixels"--1/256th of the tile width or height. (default 5) @@ -82,7 +82,7 @@ Options ### Point simplification - * -r _rate_: Rate at which dots are dropped at lower zoom levels (default 2.5) + * -r _rate_: Rate at which dots are dropped at zoom levels below basezoom (default 2.5) * -g _gamma_: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number. ### Doing more diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 3fbd4bb7..8754e4b3 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -71,11 +71,13 @@ specified, the files are all merged into the single named layer. .SS Zoom levels and resolution .RS .IP \(bu 2 -\-z \fIzoom\fP: Base (maxzoom) zoom level (default 14) +\-z \fIzoom\fP: Maxzoom: the highest zoom level for which tiles are generated (default 14) .IP \(bu 2 -\-Z \fIzoom\fP: Lowest (minzoom) zoom level (default 0) +\-Z \fIzoom\fP: Minzoom: the lowest zoom level for which tiles are generated (default 0) .IP \(bu 2 -\-d \fIdetail\fP: Detail at base zoom level (default 12, for tile resolution of 4096) +\-B \fIzoom\fP: Base zoom, the level at and above which all points are included in the tiles (default maxzoom) +.IP \(bu 2 +\-d \fIdetail\fP: Detail at max zoom level (default 12, for tile resolution of 4096) .IP \(bu 2 \-D \fIdetail\fP: Detail at lower zoom levels (default 12, for tile resolution of 4096) .IP \(bu 2 @@ -95,7 +97,7 @@ specified, the files are all merged into the single named layer. .SS Point simplification .RS .IP \(bu 2 -\-r \fIrate\fP: Rate at which dots are dropped at lower zoom levels (default 2.5) +\-r \fIrate\fP: Rate at which dots are dropped at zoom levels below basezoom (default 2.5) .IP \(bu 2 \-g \fIgamma\fP: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number. .RE @@ -168,7 +170,7 @@ coordinated with the base zoom level and dot\-dropping rate. You can use this sh calculate the appropriate marker\-width at high zoom levels to match the fraction of dots that were dropped at low zoom levels. .PP -If you used \fB\fC\-z\fR to change the base zoom level or \fB\fC\-r\fR to change the +If you used \fB\fC\-B\fR or \fB\fC\-z\fR to change the base zoom level or \fB\fC\-r\fR to change the dot\-dropping rate, replace them in the \fB\fCbasezoom\fR and \fB\fCrate\fR below. .PP .RS @@ -193,7 +195,9 @@ awk 'BEGIN { At every zoom level, line and polygon features are subjected to Douglas\-Peucker simplification to the resolution of the tile. .PP -For point features, it drops 1/2.5 of the dots for each zoom level above the base. +For point features, it drops 1/2.5 of the dots for each zoom level above the +point base zoom (which is normally the same as the \fB\fC\-z\fR max zoom, but can be +a different zoom specified with \fB\fC\-B\fR if you have precise but sparse data). I don't know why 2.5 is the appropriate number, but the densities of many different data sets fall off at about this same rate. You can use \-r to specify a different rate. .PP From 7f8eb4de83b21183e631be360f8d4bd77f335bd2 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 15 Dec 2015 15:52:08 -0800 Subject: [PATCH 05/14] Add an option to guess an appropriately dense base zoom level --- CHANGELOG.md | 5 +++ README.md | 3 +- geojson.c | 83 +++++++++++++++++++++++++++++++++++++++++++++--- man/tippecanoe.1 | 3 +- version.h | 2 +- 5 files changed, 89 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10e4a5de..0b132583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.5.0 + +* Base zoom for dot-dropping can be specified independently of + maxzoom for tiling + ## 1.4.3 * Encode numeric attributes as integers instead of floating point if possible diff --git a/README.md b/README.md index 04f86e35..71b7d77a 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,8 @@ Options * -z _zoom_: Maxzoom: the highest zoom level for which tiles are generated (default 14) * -Z _zoom_: Minzoom: the lowest zoom level for which tiles are generated (default 0) - * -B _zoom_: Base zoom, the level at and above which all points are included in the tiles (default maxzoom) + * -B _zoom_: Base zoom, the level at and above which all points are included in the tiles (default maxzoom). + If you use -Bg, it will guess a zoom level that will keep less than 50,000 features in the densest tile. * -d _detail_: Detail at max zoom level (default 12, for tile resolution of 4096) * -D _detail_: Detail at lower zoom levels (default 12, for tile resolution of 4096) * -m _detail_: Minimum detail that it will try if tiles are too big at regular detail (default 7) diff --git a/geojson.c b/geojson.c index a9e43bfb..17bb30dc 100644 --- a/geojson.c +++ b/geojson.c @@ -1035,13 +1035,84 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max close(indexfd); } - /* Copy geometries to a new file in index order */ - indexfd = open(indexname, O_RDONLY); if (indexfd < 0) { perror("reopen sorted index"); exit(EXIT_FAILURE); } + + if (basezoom < 0) { + struct index *map = mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0); + if (map == MAP_FAILED) { + perror("mmap"); + exit(EXIT_FAILURE); + } + +#define MAX_ZOOM 30 + struct tile { + unsigned x; + unsigned y; + long long count; + } tile[MAX_ZOOM + 1], max[MAX_ZOOM + 1]; + + { + int i; + for (i = 0; i <= MAX_ZOOM; i++) { + tile[i].x = tile[i].y = tile[i].count = 0; + max[i].x = max[i].y = max[i].count = 0; + } + } + + long long indices = indexpos / sizeof(struct index); + long long i; + for (i = 0; i < indices; i++) { + unsigned xx, yy; + decode(map[i].index, &xx, &yy); + + int z; + for (z = 0; z <= MAX_ZOOM; z++) { + unsigned xxx = 0, yyy = 0; + if (z != 0) { + xxx = xx >> (32 - z); + yyy = yy >> (32 - z); + } + + if (tile[z].x != xxx || tile[z].y != yyy) { + if (tile[z].count > max[z].count) { + max[z] = tile[z]; + } + + tile[z].x = xxx; + tile[z].y = yyy; + tile[z].count = 0; + } + + tile[z].count++; + } + } + + basezoom = MAX_ZOOM; + + int z; + for (z = MAX_ZOOM; z >= 0; z--) { + if (tile[z].count > max[z].count) { + max[z] = tile[z]; + } + + if (max[z].count < 50000) { + basezoom = z; + } + + // printf("%d/%u/%u %lld\n", z, max[z].x, max[z].y, max[z].count); + } + + fprintf(stderr, "Choosing a base zoom of -B%d to keep %lld features in tile %d/%u/%u.\n", basezoom, max[basezoom].count, basezoom, max[basezoom].x, max[basezoom].y); + + munmap(map, indexpos); + } + + /* Copy geometries to a new file in index order */ + struct index *index_map = mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0); if (index_map == MAP_FAILED) { perror("mmap index"); @@ -1262,7 +1333,11 @@ int main(int argc, char **argv) { break; case 'B': - basezoom = atoi(optarg); + if (strcmp(optarg, "g") == 0) { + basezoom = -2; + } else { + basezoom = atoi(optarg); + } break; case 'd': @@ -1347,7 +1422,7 @@ int main(int argc, char **argv) { exit(EXIT_FAILURE); } - if (basezoom < 0) { + if (basezoom == -1) { basezoom = maxzoom; } diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 8754e4b3..6f677bd3 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -75,7 +75,8 @@ specified, the files are all merged into the single named layer. .IP \(bu 2 \-Z \fIzoom\fP: Minzoom: the lowest zoom level for which tiles are generated (default 0) .IP \(bu 2 -\-B \fIzoom\fP: Base zoom, the level at and above which all points are included in the tiles (default maxzoom) +\-B \fIzoom\fP: Base zoom, the level at and above which all points are included in the tiles (default maxzoom). +If you use \-Bg, it will guess a zoom level that will keep less than 50,000 features in the densest tile. .IP \(bu 2 \-d \fIdetail\fP: Detail at max zoom level (default 12, for tile resolution of 4096) .IP \(bu 2 diff --git a/version.h b/version.h index 5adfc75c..955f1929 100644 --- a/version.h +++ b/version.h @@ -1 +1 @@ -#define VERSION "tippecanoe v1.4.3\n" +#define VERSION "tippecanoe v1.5.0\n" From 966bee89195613a56d04a02d47b4cd099123062d Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 15 Dec 2015 15:55:54 -0800 Subject: [PATCH 06/14] Fix bounding box calculation --- geojson.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geojson.c b/geojson.c index 17bb30dc..5ce045bb 100644 --- a/geojson.c +++ b/geojson.c @@ -1250,7 +1250,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max if (file_bbox[1] < 0) { file_bbox[1] = 0; } - if (file_bbox[3] < (1LL << 32) - 1) { + if (file_bbox[3] > (1LL << 32) - 1) { file_bbox[3] = (1LL << 32) - 1; } From eb906dfc728171bd2042862f930a16bc8f057b2c Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 15 Dec 2015 16:20:05 -0800 Subject: [PATCH 07/14] Write spaces over the reading progress message at the end --- geojson.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/geojson.c b/geojson.c index 5ce045bb..4a6f9738 100644 --- a/geojson.c +++ b/geojson.c @@ -736,6 +736,11 @@ void parse_json(json_pull *jp, const char *reading, long long *seq, long long *m /* XXX check for any non-features in the outer object */ } + + if (!quiet) { + fprintf(stderr, " \r"); + // (stderr, "Read 10000.00 million features\r", *seq / 1000000.0); + } } int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional) { From 668181724390850baceaf838a197e2ab808a502c Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 16 Dec 2015 12:13:24 -0800 Subject: [PATCH 08/14] Add an option to guess the drop rate --- README.md | 5 +++-- geojson.c | 48 +++++++++++++++++++++++++++++++++++++----------- man/tippecanoe.1 | 5 +++-- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 71b7d77a..31cc9d03 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Options * -z _zoom_: Maxzoom: the highest zoom level for which tiles are generated (default 14) * -Z _zoom_: Minzoom: the lowest zoom level for which tiles are generated (default 0) * -B _zoom_: Base zoom, the level at and above which all points are included in the tiles (default maxzoom). - If you use -Bg, it will guess a zoom level that will keep less than 50,000 features in the densest tile. + If you use -Bg, it will guess a zoom level that will keep at most 50,000 features in the densest tile. * -d _detail_: Detail at max zoom level (default 12, for tile resolution of 4096) * -D _detail_: Detail at lower zoom levels (default 12, for tile resolution of 4096) * -m _detail_: Minimum detail that it will try if tiles are too big at regular detail (default 7) @@ -83,7 +83,8 @@ Options ### Point simplification - * -r _rate_: Rate at which dots are dropped at zoom levels below basezoom (default 2.5) + * -r _rate_: Rate at which dots are dropped at zoom levels below basezoom (default 2.5). + If you use -rg, it will guess a drop rate that will keep at most 50,000 features in the densest tile. * -g _gamma_: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number. ### Doing more diff --git a/geojson.c b/geojson.c index 4a6f9738..0750fcf9 100644 --- a/geojson.c +++ b/geojson.c @@ -1046,7 +1046,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max exit(EXIT_FAILURE); } - if (basezoom < 0) { + if (basezoom < 0 || droprate < 0) { struct index *map = mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0); if (map == MAP_FAILED) { perror("mmap"); @@ -1096,22 +1096,44 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max } } - basezoom = MAX_ZOOM; - int z; for (z = MAX_ZOOM; z >= 0; z--) { if (tile[z].count > max[z].count) { max[z] = tile[z]; } - - if (max[z].count < 50000) { - basezoom = z; - } - - // printf("%d/%u/%u %lld\n", z, max[z].x, max[z].y, max[z].count); } - fprintf(stderr, "Choosing a base zoom of -B%d to keep %lld features in tile %d/%u/%u.\n", basezoom, max[basezoom].count, basezoom, max[basezoom].x, max[basezoom].y); +#define MAX_FEATURES 50000 + + if (basezoom < 0) { + basezoom = MAX_ZOOM; + + for (z = MAX_ZOOM; z >= 0; z--) { + if (max[z].count < MAX_FEATURES) { + basezoom = z; + } + + // printf("%d/%u/%u %lld\n", z, max[z].x, max[z].y, max[z].count); + } + + fprintf(stderr, "Choosing a base zoom of -B%d to keep %lld features in tile %d/%u/%u.\n", basezoom, max[basezoom].count, basezoom, max[basezoom].x, max[basezoom].y); + } + + if (droprate < 0) { + droprate = 1; + + for (z = 0; z <= basezoom; z++) { + double interval = exp(log(droprate) * (basezoom - z)); + + if (max[z].count / interval >= MAX_FEATURES) { + interval = (long double) max[z].count / MAX_FEATURES; + droprate = exp(log(interval) / (basezoom - z)); + interval = exp(log(droprate) * (basezoom - z)); + + fprintf(stderr, "Choosing a drop rate of -r%f to keep %f features in tile %d/%u/%u.\n", droprate, max[z].count / interval, z, max[z].x, max[z].y); + } + } + } munmap(map, indexpos); } @@ -1375,7 +1397,11 @@ int main(int argc, char **argv) { break; case 'r': - droprate = atof(optarg); + if (strcmp(optarg, "g") == 0) { + droprate = -2; + } else { + droprate = atof(optarg); + } break; case 'b': diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 6f677bd3..65dc9ad7 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -76,7 +76,7 @@ specified, the files are all merged into the single named layer. \-Z \fIzoom\fP: Minzoom: the lowest zoom level for which tiles are generated (default 0) .IP \(bu 2 \-B \fIzoom\fP: Base zoom, the level at and above which all points are included in the tiles (default maxzoom). -If you use \-Bg, it will guess a zoom level that will keep less than 50,000 features in the densest tile. +If you use \-Bg, it will guess a zoom level that will keep at most 50,000 features in the densest tile. .IP \(bu 2 \-d \fIdetail\fP: Detail at max zoom level (default 12, for tile resolution of 4096) .IP \(bu 2 @@ -98,7 +98,8 @@ If you use \-Bg, it will guess a zoom level that will keep less than 50,000 feat .SS Point simplification .RS .IP \(bu 2 -\-r \fIrate\fP: Rate at which dots are dropped at zoom levels below basezoom (default 2.5) +\-r \fIrate\fP: Rate at which dots are dropped at zoom levels below basezoom (default 2.5). +If you use \-rg, it will guess a drop rate that will keep at most 50,000 features in the densest tile. .IP \(bu 2 \-g \fIgamma\fP: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number. .RE From b7a411769a81664cda541a923b734bdbf227a792 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 16 Dec 2015 12:33:07 -0800 Subject: [PATCH 09/14] Don't allow <0 gamma (randomized) if basezoom or droprate not known --- geojson.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/geojson.c b/geojson.c index 0750fcf9..2138a6ef 100644 --- a/geojson.c +++ b/geojson.c @@ -1122,7 +1122,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max if (droprate < 0) { droprate = 1; - for (z = 0; z <= basezoom; z++) { + for (z = basezoom; z >= 0; z--) { double interval = exp(log(droprate) * (basezoom - z)); if (max[z].count / interval >= MAX_FEATURES) { @@ -1468,6 +1468,13 @@ int main(int argc, char **argv) { geometry_scale = 32 - (full_detail + maxzoom); + if ((basezoom < 0 || droprate < 0) && (gamma < 0)) { + // Can't use randomized (as opposed to evenly distributed) dot dropping + // if rate and base aren't known during feature reading. + gamma = 0; + fprintf(stderr, "Forcing -g0 since -B or -r is not known\n"); + } + if (outdir == NULL) { fprintf(stderr, "%s: must specify -o out.mbtiles\n", argv[0]); exit(EXIT_FAILURE); From 38fe00c7ab16c8826fc28d1af204eff88be9c07c Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 16 Dec 2015 13:09:31 -0800 Subject: [PATCH 10/14] Replicate the dot-gamma logic in the base/rate guessing code --- geojson.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/geojson.c b/geojson.c index 2138a6ef..95d37c1b 100644 --- a/geojson.c +++ b/geojson.c @@ -1058,12 +1058,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max unsigned x; unsigned y; long long count; + double gap; + unsigned long long previndex; } tile[MAX_ZOOM + 1], max[MAX_ZOOM + 1]; { int i; for (i = 0; i <= MAX_ZOOM; i++) { - tile[i].x = tile[i].y = tile[i].count = 0; + tile[i].x = tile[i].y = tile[i].count = tile[i].gap = tile[i].previndex = 0; max[i].x = max[i].y = max[i].count = 0; } } @@ -1082,6 +1084,8 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max yyy = yy >> (32 - z); } + double scale = (double) (1LL << (64 - 2 * (z + 8))); + if (tile[z].x != xxx || tile[z].y != yyy) { if (tile[z].count > max[z].count) { max[z] = tile[z]; @@ -1090,6 +1094,37 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max tile[z].x = xxx; tile[z].y = yyy; tile[z].count = 0; + tile[z].gap = 0; + tile[z].previndex = 0; + } + + // Keep in sync with write_tile() + if (gamma > 0) { + if (tile[z].gap > 0) { + if (map[i].index == tile[z].previndex) { + continue; // Exact duplicate: can't fulfil the gap requirement + } + + if (exp(log((map[i].index - tile[z].previndex) / scale) * gamma) >= tile[z].gap) { + // Dot is further from the previous than the nth root of the gap, + // so produce it, and choose a new gap at the next point. + tile[z].gap = 0; + } else { + continue; + } + } else { + tile[z].gap = (map[i].index - tile[z].previndex) / scale; + + if (tile[z].gap == 0) { + continue; // Exact duplicate: skip + } else if (tile[z].gap < 1) { + continue; // Narrow dot spacing: need to stretch out + } else { + tile[z].gap = 0; // Wider spacing than minimum: so pass through unchanged + } + } + + tile[z].previndex = map[i].index; } tile[z].count++; From f9d1e87360949df25881d1d21736273fe6382e47 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 16 Dec 2015 16:18:16 -0800 Subject: [PATCH 11/14] Heuristics for basezoom and droprate if no tiles are naturally small enough --- geojson.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/geojson.c b/geojson.c index 95d37c1b..a9615673 100644 --- a/geojson.c +++ b/geojson.c @@ -1140,6 +1140,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max #define MAX_FEATURES 50000 + int obasezoom = basezoom; if (basezoom < 0) { basezoom = MAX_ZOOM; @@ -1154,10 +1155,36 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max fprintf(stderr, "Choosing a base zoom of -B%d to keep %lld features in tile %d/%u/%u.\n", basezoom, max[basezoom].count, basezoom, max[basezoom].x, max[basezoom].y); } - if (droprate < 0) { + if (obasezoom < 0 && basezoom > maxzoom) { + fprintf(stderr, "Couldn't find a suitable base zoom. Working from the other direction.\n"); + if (gamma == 0) { + fprintf(stderr, "You might want to try -g1 to limit near-duplicates.\n"); + } + + if (droprate < 0) { + if (maxzoom == 0) { + droprate = 2.5; + } else { + droprate = exp(log((long double) max[0].count / max[maxzoom].count) / (maxzoom)); + fprintf(stderr, "Choosing a drop rate of -r%f to get from %lld to %lld in %d zooms\n", droprate, max[maxzoom].count, max[0].count, maxzoom); + } + } + + basezoom = 0; + for (z = 0; z <= maxzoom; z++) { + // printf("%d/%u/%u has %lld ", z, max[z].x, max[z].y, max[z].count); + double zoomdiff = log((long double) max[z].count / MAX_FEATURES) / log(droprate); + // printf("which implies that base zoom should be %f\n", zoomdiff + z); + if (zoomdiff + z > basezoom) { + basezoom = ceil(zoomdiff + z); + } + } + + fprintf(stderr, "Choosing a base zoom of -B%d to keep %f features in tile %d/%u/%u.\n", basezoom, max[maxzoom].count * exp(log(droprate) * (maxzoom - basezoom)), maxzoom, max[maxzoom].x, max[maxzoom].y); + } else if (droprate < 0) { droprate = 1; - for (z = basezoom; z >= 0; z--) { + for (z = basezoom - 1; z >= 0; z--) { double interval = exp(log(droprate) * (basezoom - z)); if (max[z].count / interval >= MAX_FEATURES) { From acd444443996ec2bfa485a272771aee3375957d3 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 17 Dec 2015 12:33:52 -0800 Subject: [PATCH 12/14] Allow specifying a marker width for automatic basezoom guessing --- README.md | 2 ++ geojson.c | 35 ++++++++++++++++++++++++++--------- man/tippecanoe.1 | 2 ++ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 31cc9d03..4f71c887 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ Options * -Z _zoom_: Minzoom: the lowest zoom level for which tiles are generated (default 0) * -B _zoom_: Base zoom, the level at and above which all points are included in the tiles (default maxzoom). If you use -Bg, it will guess a zoom level that will keep at most 50,000 features in the densest tile. + You can also specify a marker-width with -Bg*width* to allow fewer features in the densest tile to + compensate for the larger marker. * -d _detail_: Detail at max zoom level (default 12, for tile resolution of 4096) * -D _detail_: Detail at lower zoom levels (default 12, for tile resolution of 4096) * -m _detail_: Minimum detail that it will try if tiles are too big at regular detail (default 7) diff --git a/geojson.c b/geojson.c index a9615673..5ef17166 100644 --- a/geojson.c +++ b/geojson.c @@ -743,7 +743,7 @@ void parse_json(json_pull *jp, const char *reading, long long *seq, long long *m } } -int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional) { +int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional) { int ret = EXIT_SUCCESS; char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; @@ -1058,6 +1058,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max unsigned x; unsigned y; long long count; + long long fullcount; double gap; unsigned long long previndex; } tile[MAX_ZOOM + 1], max[MAX_ZOOM + 1]; @@ -1065,8 +1066,8 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max { int i; for (i = 0; i <= MAX_ZOOM; i++) { - tile[i].x = tile[i].y = tile[i].count = tile[i].gap = tile[i].previndex = 0; - max[i].x = max[i].y = max[i].count = 0; + tile[i].x = tile[i].y = tile[i].count = tile[i].fullcount = tile[i].gap = tile[i].previndex = 0; + max[i].x = max[i].y = max[i].count = max[i].fullcount = 0; } } @@ -1094,10 +1095,13 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max tile[z].x = xxx; tile[z].y = yyy; tile[z].count = 0; + tile[z].fullcount = 0; tile[z].gap = 0; tile[z].previndex = 0; } + tile[z].fullcount++; + // Keep in sync with write_tile() if (gamma > 0) { if (tile[z].gap > 0) { @@ -1138,14 +1142,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max } } -#define MAX_FEATURES 50000 + int max_features = 50000 / (basezoom_marker_width * basezoom_marker_width); int obasezoom = basezoom; if (basezoom < 0) { basezoom = MAX_ZOOM; for (z = MAX_ZOOM; z >= 0; z--) { - if (max[z].count < MAX_FEATURES) { + if (max[z].count < max_features) { basezoom = z; } @@ -1173,7 +1177,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max basezoom = 0; for (z = 0; z <= maxzoom; z++) { // printf("%d/%u/%u has %lld ", z, max[z].x, max[z].y, max[z].count); - double zoomdiff = log((long double) max[z].count / MAX_FEATURES) / log(droprate); + double zoomdiff = log((long double) max[z].count / max_features) / log(droprate); // printf("which implies that base zoom should be %f\n", zoomdiff + z); if (zoomdiff + z > basezoom) { basezoom = ceil(zoomdiff + z); @@ -1187,8 +1191,8 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max for (z = basezoom - 1; z >= 0; z--) { double interval = exp(log(droprate) * (basezoom - z)); - if (max[z].count / interval >= MAX_FEATURES) { - interval = (long double) max[z].count / MAX_FEATURES; + if (max[z].count / interval >= max_features) { + interval = (long double) max[z].count / max_features; droprate = exp(log(interval) / (basezoom - z)); interval = exp(log(droprate) * (basezoom - z)); @@ -1385,6 +1389,7 @@ int main(int argc, char **argv) { int maxzoom = 14; int minzoom = 0; int basezoom = -1; + double basezoom_marker_width = 1; int force = 0; double droprate = 2.5; double gamma = 0; @@ -1424,8 +1429,20 @@ int main(int argc, char **argv) { case 'B': if (strcmp(optarg, "g") == 0) { basezoom = -2; + basezoom_marker_width = 1; + } else if (optarg[0] == 'g') { + basezoom = -2; + basezoom_marker_width = atof(optarg + 1); + if (basezoom_marker_width == 0) { + fprintf(stderr, "%s: Must specify marker width >0 with -Bg\n", argv[0]); + exit(EXIT_FAILURE); + } } else { basezoom = atoi(optarg); + if (basezoom == 0 && strcmp(optarg, "0") != 0) { + fprintf(stderr, "%s: Couldn't understand -B%s\n", argv[0], optarg); + exit(EXIT_FAILURE); + } } break; @@ -1549,7 +1566,7 @@ int main(int argc, char **argv) { sqlite3 *outdb = mbtiles_open(outdir, argv); int ret = EXIT_SUCCESS; - ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, basezoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional); + ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional); mbtiles_close(outdb, argv); diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index 65dc9ad7..28d1828b 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -77,6 +77,8 @@ specified, the files are all merged into the single named layer. .IP \(bu 2 \-B \fIzoom\fP: Base zoom, the level at and above which all points are included in the tiles (default maxzoom). If you use \-Bg, it will guess a zoom level that will keep at most 50,000 features in the densest tile. +You can also specify a marker\-width with \-Bg\fIwidth\fP to allow fewer features in the densest tile to +compensate for the larger marker. .IP \(bu 2 \-d \fIdetail\fP: Detail at max zoom level (default 12, for tile resolution of 4096) .IP \(bu 2 From 4fe1a950f5b936c8b826a9c6692cf5adbff85d9d Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 17 Dec 2015 17:16:04 -0800 Subject: [PATCH 13/14] Calculate effective basezoom and droprate, taking gamma into account --- geojson.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/geojson.c b/geojson.c index 5ef17166..6248b570 100644 --- a/geojson.c +++ b/geojson.c @@ -1176,9 +1176,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max basezoom = 0; for (z = 0; z <= maxzoom; z++) { - // printf("%d/%u/%u has %lld ", z, max[z].x, max[z].y, max[z].count); double zoomdiff = log((long double) max[z].count / max_features) / log(droprate); - // printf("which implies that base zoom should be %f\n", zoomdiff + z); if (zoomdiff + z > basezoom) { basezoom = ceil(zoomdiff + z); } @@ -1201,6 +1199,36 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max } } + if (gamma > 0) { + int effective = 0; + + for (z = 0; z < maxzoom; z++) { + if (max[z].count < max[z].fullcount) { + effective = z + 1; + } + } + + if (effective == 0) { + fprintf(stderr, "With gamma, effective base zoom is 0, so no effective drop rate\n"); + } else { + double interval_0 = exp(log(droprate) * (basezoom - 0)); + double interval_eff = exp(log(droprate) * (basezoom - effective)); + if (effective > basezoom) { + interval_eff = 1; + } + + double scaled_0 = max[0].count / interval_0; + double scaled_eff = max[effective].count / interval_eff; + + double rate_at_0 = scaled_0 / max[0].fullcount; + double rate_at_eff = scaled_eff / max[effective].fullcount; + + double eff_drop = exp(log(rate_at_eff / rate_at_0) / (effective - 0)); + + fprintf(stderr, "With gamma, effective base zoom of %d, effective drop rate of %f\n", effective, eff_drop); + } + } + munmap(map, indexpos); } From fd682b66a8f93b7050ae39853816c99d5e72afcb Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 18 Dec 2015 11:28:00 -0800 Subject: [PATCH 14/14] Mention base zoom and drop rate calculation in changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b132583..52a6daa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ ## 1.5.0 * Base zoom for dot-dropping can be specified independently of - maxzoom for tiling + maxzoom for tiling. +* Tippecanoe can calculate a base zoom and drop rate for you. ## 1.4.3