diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 16:35:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Scale the tile size limit up with the multiplier at low zooms (#192)

Browse Source 
* Scale the tile size limit up with the multiplier at low zooms

* Add a test to demonstrate that high zoom tiles can't be extra large

* Update changelog and version

* Fail more cleanly when a tile can't be made small enough

* Guard against a cluster where the start marker has been dropped

* Look harder for a working feature interval instead of giving up

* That change to the drop-smallest logic changed a test output

* Update changelog

* Add explanatory comment
This commit is contained in:
Erica Fischer
2024-01-31 10:23:05 -08:00
committed by GitHub
parent ca1aa58a11
commit 6a2bce8164
9 changed files with 22368 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tile.cpp
+75 -20
View File
@@ -361,7 +361,7 @@ std::vector<std::vector<coalesce>> assemble_multiplier_clusters(std::vector<coal
}
}
if (is_cluster_start) {
if (is_cluster_start || clusters.size() == 0) {
clusters.push_back(std::vector<coalesce>());
}
@@ -1358,9 +1358,45 @@ unsigned long long choose_mingap(std::vector<unsigned long long> const &indices,
return top;
}
long long choose_minextent(std::vector<long long> &extents, double f) {
// This function is called to choose the new "extent" threshold to try when a tile exceeds the
// tile size limit or feature limit and `--drop-smallest-as-needed` or `--coalesce-smallest-as-needed`
// has been set.
//
// The "extents" are the areas of the polygon features or the pseudo-areas associated with the
// linestring or point features that were examined for inclusion in the most recent
// iteration of this tile. (This includes features that were dropped because they were below
// the previous size threshold, but not features that were dropped by fractional point dropping).
// The extents are placed in order by the sort, from smallest to largest.
//
// The `fraction` is the proportion of these features that tippecanoe thinks should be retained to
// to make the tile small enough now. Because the extents are sorted from smallest to largest,
// the smallest extent threshold that will retain that fraction of features is found `fraction`
// distance from the end of the list, or at element `(1 - fraction) * (size() - 1)`.
//
// However, the extent found there may be the same extent that was used in the last iteration!
//
// (The "existing_extent" is the extent threshold that selected these features in the recent
// iteration. It is 0 the first time a tile is attempted, and gets higher on successive iterations
// as tippecanoe restricts the features to be kept to larger and larger features.)
//
// The features that are kept are those with a size >= the existing_extent, so if there are a large
// number of features with identical small areas, the new guess may not exclude enough features
// to actually choose a new threshold larger than the previous threshold.
//
// To address this, the array index `ix` of the new chosen extent is incremented toward the end
// of the list, until the possibilities run out or something higher than the old extent is found.
// If there are no higher extents available, the tile has already been reduced as much as possible
// and tippecanoe will exit with an error.
long long choose_minextent(std::vector<long long> &extents, double f, long long existing_extent) {
std::sort(extents.begin(), extents.end());
return extents[(extents.size() - 1) * (1 - f)];
size_t ix = (extents.size() - 1) * (1 - f);
while (ix + 1 < extents.size() && extents[ix] == existing_extent) {
ix++;
}
return extents[ix];
}
struct write_tile_args {
@@ -1383,6 +1419,8 @@ struct write_tile_args {
std::atomic<unsigned> *midy = NULL;
int maxzoom = 0;
int minzoom = 0;
int basezoom = 0;
double droprate = 0;
int full_detail = 0;
int low_detail = 0;
double simplification = 0;
@@ -1960,6 +1998,20 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
double mingap_fraction = 1;
double minextent_fraction = 1;
// allow larger tile sizes at low zooms when the retain-points-multiplier
// is intended to allow more points through. scale back down toward a
// tile size multiple of 1 at basezoom and beyond
size_t scaled_max_tile_size = max_tile_size;
double regular_retention = 1 / exp(log(arg->droprate) * (arg->basezoom - z));
if (regular_retention > 1) {
regular_retention = 1;
}
double multiplier_retention = 1 / exp(log(arg->droprate) * (arg->basezoom - z)) * retain_points_multiplier;
if (multiplier_retention > 1) {
multiplier_retention = 1;
}
scaled_max_tile_size *= multiplier_retention / regular_retention;
static std::atomic<double> oprogress(0);
long long og = *geompos_in;
@@ -2291,7 +2343,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
if (sf.geometry.size() > 0) {
if (partials.size() > max_tile_size) {
if (partials.size() > scaled_max_tile_size) {
// Even being maximally conservative, each feature is still going to be
// at least one byte in the output tile, so this can't possibly work.
skipped++;
@@ -2817,7 +2869,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
continue;
} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
minextent_fraction = minextent_fraction * max_tile_features / totalsize * 0.75;
long long m = choose_minextent(extents, minextent_fraction);
long long m = choose_minextent(extents, minextent_fraction, minextent);
if (m != minextent) {
minextent = m;
if (minextent > arg->minextent_out) {
@@ -2863,7 +2915,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
compressed = pbf;
}
if (compressed.size() > max_tile_size && !prevent[P_KILOBYTE_LIMIT]) {
if (compressed.size() > scaled_max_tile_size && !prevent[P_KILOBYTE_LIMIT]) {
// Estimate how big it really should have been compressed
// from how many features were kept vs skipped for already being
// over the threshold
@@ -2876,9 +2928,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (!quiet) {
if (skipped > 0) {
fprintf(stderr, "tile %d/%u/%u size is %lld (probably really %lld) with detail %d, >%zu \n", z, tx, ty, (long long) compressed.size(), (long long) (compressed.size() * kept_adjust), line_detail, max_tile_size);
fprintf(stderr, "tile %d/%u/%u size is %lld (probably really %lld) with detail %d, >%zu \n", z, tx, ty, (long long) compressed.size(), (long long) (compressed.size() * kept_adjust), line_detail, scaled_max_tile_size);
} else {
fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >%zu \n", z, tx, ty, (long long) compressed.size(), line_detail, max_tile_size);
fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >%zu \n", z, tx, ty, (long long) compressed.size(), line_detail, scaled_max_tile_size);
}
}
@@ -2899,7 +2951,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
line_detail++; // to keep it the same when the loop decrements it
} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
mingap_fraction = mingap_fraction * max_tile_size / (kept_adjust * compressed.size()) * 0.90;
mingap_fraction = mingap_fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.90;
unsigned long long mg = choose_mingap(indices, mingap_fraction);
if (mg <= mingap) {
double nmg = (mingap + 1) * 1.5;
@@ -2924,8 +2976,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
line_detail++;
} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
minextent_fraction = minextent_fraction * max_tile_size / (kept_adjust * compressed.size()) * 0.75;
long long m = choose_minextent(extents, minextent_fraction);
minextent_fraction = minextent_fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.75;
long long m = choose_minextent(extents, minextent_fraction, minextent);
if (m != minextent) {
minextent = m;
if (minextent > arg->minextent_out) {
@@ -2942,7 +2994,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// The 95% is a guess to avoid too many retries
// and probably actually varies based on how much duplicated metadata there is
fraction = fraction * max_tile_size / (kept_adjust * compressed.size()) * 0.95;
fraction = fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.95;
if (!quiet) {
fprintf(stderr, "Going to try keeping %0.2f%% of the features to make it fit\n", fraction * 100);
}
@@ -2992,6 +3044,7 @@ struct task {
void *run_thread(void *vargs) {
write_tile_args *arg = (write_tile_args *) vargs;
struct task *task;
int *err_or_null = NULL;
for (task = arg->tasks; task != NULL; task = task->next) {
int j = task->fileno;
@@ -3055,12 +3108,6 @@ void *run_thread(void *vargs) {
long long len = write_tile(&dc, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos);
if (len < 0) {
int *err = &arg->err;
*err = z - 1;
return err;
}
if (pthread_mutex_lock(&var_lock) != 0) {
perror("pthread_mutex_lock");
exit(EXIT_PTHREAD);
@@ -3090,6 +3137,12 @@ void *run_thread(void *vargs) {
perror("pthread_mutex_unlock");
exit(EXIT_PTHREAD);
}
if (len < 0) {
err_or_null = &arg->err;
*err_or_null = z - 1;
break;
}
}
if (arg->pass == 1) {
@@ -3115,10 +3168,10 @@ void *run_thread(void *vargs) {
}
arg->running--;
return NULL;
return err_or_null;
}
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos) {
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, int basezoom, double droprate) {
last_progress = 0;
// The existing layermaps are one table per input thread.
@@ -3301,6 +3354,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<
args[thread].midy = midy; // locked with var_lock
args[thread].maxzoom = maxzoom;
args[thread].minzoom = minzoom;
args[thread].basezoom = basezoom;
args[thread].droprate = droprate;
args[thread].full_detail = full_detail;
args[thread].low_detail = low_detail;
args[thread].most = &most; // locked with var_lock