diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Estimate tile complexity; don't try truncating when unlikely to work

Browse Source
This commit is contained in:
Erica Fischer
2024-07-30 15:26:09 -07:00
parent 049c24c754
commit 81436a783d
2 changed files with 30 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+5 -2
View File
@@ -2219,8 +2219,8 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
std::atomic<long long> geompos(0); std::atomic<long long> geompos(0);
/* initial tile is normally 0/0/0 but can be iz/ix/iy if limited to one tile */ /* initial tile is normally 0/0/0 but can be iz/ix/iy if limited to one tile */
long long vertex_count = 0; long long estimated_complexity = 0; // to be replaced after writing the data
fwrite_check(&vertex_count, sizeof(vertex_count), 1, geomfile, &geompos, fname); fwrite_check(&estimated_complexity, sizeof(estimated_complexity), 1, geomfile, &geompos, fname);
serialize_int(geomfile, iz, &geompos, fname); serialize_int(geomfile, iz, &geompos, fname);
serialize_uint(geomfile, ix, &geompos, fname); serialize_uint(geomfile, ix, &geompos, fname);
serialize_uint(geomfile, iy, &geompos, fname); serialize_uint(geomfile, iy, &geompos, fname);
@@ -2230,6 +2230,9 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
/* end of tile */ /* end of tile */
serialize_ulong_long(geomfile, 0, &geompos, fname); // EOF serialize_ulong_long(geomfile, 0, &geompos, fname); // EOF
estimated_complexity = geompos;
pwrite(fileno(geomfile), &estimated_complexity, sizeof(estimated_complexity), 0);
if (fclose(geomfile) != 0) { if (fclose(geomfile) != 0) {
perror("fclose geom"); perror("fclose geom");
exit(EXIT_CLOSE); exit(EXIT_CLOSE);
tile.cpp
+25 -9
View File
@@ -532,8 +532,8 @@ static void rewrite(serial_feature const &osf, int z, int nextzoom, int maxzoom,
within[j] = true; within[j] = true;
start_geompos[j] = geompos[j]; // no competition between threads start_geompos[j] = geompos[j]; // no competition between threads
long long vertex_count = 0; long long estimated_complexity = 0; // placeholder, to be filled in later
fwrite_check(&vertex_count, sizeof(vertex_count), 1, geomfile[j]->fp, &geompos[j], fname); fwrite_check(&estimated_complexity, sizeof(estimated_complexity), 1, geomfile[j]->fp, &geompos[j], fname);
serialize_int(geomfile[j]->fp, nextzoom, &geompos[j], fname); serialize_int(geomfile[j]->fp, nextzoom, &geompos[j], fname);
serialize_uint(geomfile[j]->fp, tx * span + xo, &geompos[j], fname); serialize_uint(geomfile[j]->fp, tx * span + xo, &geompos[j], fname);
serialize_uint(geomfile[j]->fp, ty * span + yo, &geompos[j], fname); serialize_uint(geomfile[j]->fp, ty * span + yo, &geompos[j], fname);
@@ -1514,7 +1514,7 @@ bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features
return false; // did not drop because nothing could be found to accumulate attributes onto return false; // did not drop because nothing could be found to accumulate attributes onto
} }
long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::vector<std::string> const &unidecode_data) { long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::vector<std::string> const &unidecode_data, long long estimated_complexity) {
double merge_fraction = 1; double merge_fraction = 1;
double mingap_fraction = 1; double mingap_fraction = 1;
double minextent_fraction = 1; double minextent_fraction = 1;
@@ -1554,9 +1554,15 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// and which loops if everything doesn't fit rather than trying to drop or union features. // and which loops if everything doesn't fit rather than trying to drop or union features.
// XXX special case for points below basezoom // XXX special case for points below basezoom
first_detail = 30 - z; // empirical estimate from ne_10m_admin_0_countries, CPAD units, Cal fires.
second_detail = detail; // only try to make an overzoomable final tile if it seems like it might work
trying_to_stop_early = true; long long estimated_output_tile_size = 0.6693 * estimated_complexity - 3.36e+04;
if (estimated_output_tile_size < (long long) (2 * max_tile_size)) {
first_detail = 30 - z;
second_detail = detail;
trying_to_stop_early = true;
}
} }
bool first_time = true; bool first_time = true;
@@ -2048,9 +2054,15 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
for (int j = 0; j < child_shards; j++) { for (int j = 0; j < child_shards; j++) {
if (within[j]) { if (within[j]) {
long long estimated_complexity_out = geompos[j] - start_geompos[j];
geomfile[j]->serialize_long_long(0, &geompos[j], fname); // EOF geomfile[j]->serialize_long_long(0, &geompos[j], fname); // EOF
geomfile[j]->end(&geompos[j], fname); geomfile[j]->end(&geompos[j], fname);
within[j] = false; within[j] = false;
if (additional[A_TRUNCATE_ZOOMS]) {
pwrite(fileno(geomfile[j]->fp), &estimated_complexity_out, sizeof(estimated_complexity_out), start_geompos[j]);
}
} }
} }
@@ -2498,6 +2510,10 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
compressed = pbf; compressed = pbf;
} }
if (trying_to_stop_early && line_detail == first_detail) {
// printf("%lld %zu\n", estimated_complexity, compressed.size());
}
if (compressed.size() > scaled_max_tile_size && !prevent[P_KILOBYTE_LIMIT]) { if (compressed.size() > scaled_max_tile_size && !prevent[P_KILOBYTE_LIMIT]) {
// Estimate how big it really should have been compressed // Estimate how big it really should have been compressed
// from how many features were kept vs skipped for already being // from how many features were kept vs skipped for already being
@@ -2672,8 +2688,8 @@ void *run_thread(void *vargs) {
// These z/x/y are uncompressed so we can seek to the start of the // These z/x/y are uncompressed so we can seek to the start of the
// compressed feature data that immediately follows. // compressed feature data that immediately follows.
long long vertex_count; long long estimated_complexity;
if (dc.fread(&vertex_count, sizeof(vertex_count), 1, &geompos) != 1) { if (dc.fread(&estimated_complexity, sizeof(estimated_complexity), 1, &geompos) != 1) {
break; break;
} }
if (!dc.deserialize_int(&z, &geompos)) { if (!dc.deserialize_int(&z, &geompos)) {
@@ -2697,7 +2713,7 @@ exit(EXIT_IMPOSSIBLE);
// fprintf(stderr, "%d/%u/%u\n", z, x, y); // fprintf(stderr, "%d/%u/%u\n", z, x, y);
long long len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, (*arg->unidecode_data)); long long len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, (*arg->unidecode_data), estimated_complexity);
if (pthread_mutex_lock(&var_lock) != 0) { if (pthread_mutex_lock(&var_lock) != 0) {
perror("pthread_mutex_lock"); perror("pthread_mutex_lock");