From aec0d2692b97c7bf72b1583ca560d4ea0ba5c66f Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Wed, 31 Jul 2024 22:29:07 -0700 Subject: [PATCH] Don't double-count feature dropping when the zoom level is retried --- tests/muni/out/-Z11_-z13_-M10000.json | 2 +- tests/muni/out/-Z11_-z13_-M10000_-aG.json | 2 +- tests/muni/out/-Z11_-z13_-M10000_-pd.json | 2 +- .../-ae_-zg_-M5000_--force-feature-limit.json | 2 +- ...5_-M5000_--coalesce-densest-as-needed.json | 2 +- ..._-M5000_--coalesce-fraction-as-needed.json | 2 +- ..._-M5000_--coalesce-smallest-as-needed.json | 2 +- ...st-as-needed_--truncate-zooms_-M25000.json | 2 +- tile.cpp | 39 +++++++++++++------ tile.hpp | 5 ++- 10 files changed, 39 insertions(+), 21 deletions(-) diff --git a/tests/muni/out/-Z11_-z13_-M10000.json b/tests/muni/out/-Z11_-z13_-M10000.json index 28c4072e..6197d324 100644 --- a/tests/muni/out/-Z11_-z13_-M10000.json +++ b/tests/muni/out/-Z11_-z13_-M10000.json @@ -9,7 +9,7 @@ "maxzoom": "13", "minzoom": "11", "name": "tests/muni/out/-Z11_-z13_-M10000.json.check.mbtiles", -"strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":18540,\"detail_reduced\":4,\"tile_size_desired\":10887},{\"dropped_by_rate\":6391,\"detail_reduced\":3,\"tile_size_desired\":10721},{}]", +"strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":4080,\"detail_reduced\":4,\"tile_size_desired\":10887},{\"dropped_by_rate\":2974,\"detail_reduced\":3,\"tile_size_desired\":10721},{}]", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/muni/out/-Z11_-z13_-M10000_-aG.json b/tests/muni/out/-Z11_-z13_-M10000_-aG.json index 2222049b..83d67dd0 100644 --- a/tests/muni/out/-Z11_-z13_-M10000_-aG.json +++ b/tests/muni/out/-Z11_-z13_-M10000_-aG.json @@ -9,7 +9,7 @@ "maxzoom": "13", "minzoom": "11", "name": "tests/muni/out/-Z11_-z13_-M10000_-aG.json.check.mbtiles", -"strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":14925,\"dropped_by_gamma\":53,\"detail_reduced\":3,\"tile_size_desired\":10887},{\"dropped_by_rate\":6391,\"dropped_by_gamma\":190,\"detail_reduced\":3,\"tile_size_desired\":10721},{}]", +"strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":4080,\"dropped_by_gamma\":17,\"detail_reduced\":3,\"tile_size_desired\":10887},{\"dropped_by_rate\":2974,\"dropped_by_gamma\":85,\"detail_reduced\":3,\"tile_size_desired\":10721},{}]", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/muni/out/-Z11_-z13_-M10000_-pd.json b/tests/muni/out/-Z11_-z13_-M10000_-pd.json index 5eab06e8..c729a45e 100644 --- a/tests/muni/out/-Z11_-z13_-M10000_-pd.json +++ b/tests/muni/out/-Z11_-z13_-M10000_-pd.json @@ -9,7 +9,7 @@ "maxzoom": "13", "minzoom": "11", "name": "tests/muni/out/-Z11_-z13_-M10000_-pd.json.check.mbtiles", -"strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":7695,\"dropped_as_needed\":212,\"tile_size_desired\":10887},{\"dropped_by_rate\":4113,\"dropped_as_needed\":225,\"tile_size_desired\":10721},{}]", +"strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":4080,\"dropped_as_needed\":212,\"tile_size_desired\":10887},{\"dropped_by_rate\":2974,\"dropped_as_needed\":225,\"tile_size_desired\":10721},{}]", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--force-feature-limit.json b/tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--force-feature-limit.json index bb126a84..0aa9f622 100644 --- a/tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--force-feature-limit.json +++ b/tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--force-feature-limit.json @@ -9,7 +9,7 @@ "maxzoom": "5", "minzoom": "0", "name": "tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--force-feature-limit.json.check.mbtiles", -"strategies": "[{\"dropped_as_needed\":324,\"tile_size_desired\":39239},{\"dropped_as_needed\":160,\"tiny_polygons\":2,\"tile_size_desired\":25163},{\"dropped_as_needed\":124,\"tiny_polygons\":2,\"tile_size_desired\":21214},{\"dropped_as_needed\":80,\"tile_size_desired\":10758},{\"dropped_as_needed\":12,\"tile_size_desired\":6601},{\"tiny_polygons\":2}]", +"strategies": "[{\"dropped_as_needed\":165,\"tile_size_desired\":39239},{\"dropped_as_needed\":160,\"tiny_polygons\":1,\"tile_size_desired\":25163},{\"dropped_as_needed\":124,\"tiny_polygons\":1,\"tile_size_desired\":21214},{\"dropped_as_needed\":80,\"tile_size_desired\":10758},{\"dropped_as_needed\":12,\"tile_size_desired\":6601},{\"tiny_polygons\":2}]", "tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}", "type": "overlay", "version": "2" diff --git a/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-densest-as-needed.json b/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-densest-as-needed.json index f202b41e..5deb0096 100644 --- a/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-densest-as-needed.json +++ b/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-densest-as-needed.json @@ -9,7 +9,7 @@ "maxzoom": "5", "minzoom": "0", "name": "tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-densest-as-needed.json.check.mbtiles", -"strategies": "[{\"coalesced_as_needed\":10005,\"tiny_polygons\":2,\"tile_size_desired\":39268},{\"coalesced_as_needed\":213,\"tile_size_desired\":25177},{\"coalesced_as_needed\":197,\"tile_size_desired\":21227},{\"coalesced_as_needed\":150,\"tile_size_desired\":10748},{\"coalesced_as_needed\":60,\"tile_size_desired\":6588},{\"tiny_polygons\":1}]", +"strategies": "[{\"coalesced_as_needed\":175,\"tiny_polygons\":1,\"tile_size_desired\":39268},{\"coalesced_as_needed\":213,\"tile_size_desired\":25177},{\"coalesced_as_needed\":197,\"tile_size_desired\":21227},{\"coalesced_as_needed\":150,\"tile_size_desired\":10748},{\"coalesced_as_needed\":60,\"tile_size_desired\":6588},{\"tiny_polygons\":1}]", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-fraction-as-needed.json b/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-fraction-as-needed.json index cca0f2ec..92d1641d 100644 --- a/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-fraction-as-needed.json +++ b/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-fraction-as-needed.json @@ -9,7 +9,7 @@ "maxzoom": "5", "minzoom": "0", "name": "tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-fraction-as-needed.json.check.mbtiles", -"strategies": "[{\"coalesced_as_needed\":528,\"detail_reduced\":2,\"tiny_polygons\":2,\"tile_size_desired\":39240},{\"coalesced_as_needed\":216,\"tile_size_desired\":25160},{\"coalesced_as_needed\":213,\"tile_size_desired\":21223},{\"coalesced_as_needed\":204,\"tile_size_desired\":10749},{\"coalesced_as_needed\":126,\"tile_size_desired\":6591},{\"tiny_polygons\":1}]", +"strategies": "[{\"coalesced_as_needed\":176,\"detail_reduced\":2,\"tiny_polygons\":1,\"tile_size_desired\":39240},{\"coalesced_as_needed\":216,\"tile_size_desired\":25160},{\"coalesced_as_needed\":213,\"tile_size_desired\":21223},{\"coalesced_as_needed\":204,\"tile_size_desired\":10749},{\"coalesced_as_needed\":126,\"tile_size_desired\":6591},{\"tiny_polygons\":1}]", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-smallest-as-needed.json b/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-smallest-as-needed.json index 23b7d56f..573722b7 100644 --- a/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-smallest-as-needed.json +++ b/tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-smallest-as-needed.json @@ -9,7 +9,7 @@ "maxzoom": "5", "minzoom": "0", "name": "tests/ne_110m_admin_0_countries/out/-z5_-M5000_--coalesce-smallest-as-needed.json.check.mbtiles", -"strategies": "[{\"coalesced_as_needed\":7519,\"tiny_polygons\":2,\"tile_size_desired\":39240},{\"coalesced_as_needed\":442,\"tile_size_desired\":25160},{\"coalesced_as_needed\":214,\"tiny_polygons\":1,\"tile_size_desired\":21223},{\"coalesced_as_needed\":179,\"tile_size_desired\":10749},{\"coalesced_as_needed\":73,\"tile_size_desired\":6591},{\"tiny_polygons\":1}]", +"strategies": "[{\"coalesced_as_needed\":175,\"tiny_polygons\":1,\"tile_size_desired\":39240},{\"coalesced_as_needed\":212,\"tile_size_desired\":25160},{\"coalesced_as_needed\":214,\"tiny_polygons\":1,\"tile_size_desired\":21223},{\"coalesced_as_needed\":179,\"tile_size_desired\":10749},{\"coalesced_as_needed\":73,\"tile_size_desired\":6591},{\"tiny_polygons\":1}]", "type": "overlay", "version": "2" }, "features": [ diff --git a/tests/tl_2022_11_tract/out/-z14_-Z12_--coalesce-densest-as-needed_--truncate-zooms_-M25000.json b/tests/tl_2022_11_tract/out/-z14_-Z12_--coalesce-densest-as-needed_--truncate-zooms_-M25000.json index 47e95c52..5fefc57f 100644 --- a/tests/tl_2022_11_tract/out/-z14_-Z12_--coalesce-densest-as-needed_--truncate-zooms_-M25000.json +++ b/tests/tl_2022_11_tract/out/-z14_-Z12_--coalesce-densest-as-needed_--truncate-zooms_-M25000.json @@ -9,7 +9,7 @@ "maxzoom": "14", "minzoom": "12", "name": "tests/tl_2022_11_tract/out/-z14_-Z12_--coalesce-densest-as-needed_--truncate-zooms_-M25000.json.check.mbtiles", -"strategies": "[{},{},{},{},{},{},{},{},{},{},{},{},{\"coalesced_as_needed\":146,\"tile_size_desired\":39190},{},{}]", +"strategies": "[{},{},{},{},{},{},{},{},{},{},{},{},{\"coalesced_as_needed\":73,\"tile_size_desired\":39190},{},{}]", "type": "overlay", "version": "2" }, "features": [ diff --git a/tile.cpp b/tile.cpp index bb5f51a0..92cdee28 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1508,7 +1508,7 @@ struct layer_features { size_t multiplier_cluster_size = 0; // The feature count of the current multiplier cluster }; -bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features &layer, serial_feature &sf, std::vector> *layer_unmaps, size_t &multiplier_seq, atomic_strategy *strategy, bool &drop_rest, std::unordered_map const *attribute_accum) { +bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features &layer, serial_feature &sf, std::vector> *layer_unmaps, size_t &multiplier_seq, strategy &strategy, bool &drop_rest, std::unordered_map const *attribute_accum) { ssize_t which_serial_feature; if (find_feature_to_accumulate_onto(layer.features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) { @@ -1519,7 +1519,7 @@ bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features return false; // converted rather than dropped } else { preserve_attributes(attribute_accum, sf, layer.features[which_serial_feature]); - strategy->dropped_as_needed++; + strategy.dropped_as_needed++; drop_rest = true; return true; // dropped } @@ -1534,7 +1534,7 @@ void skip_tile(decompressor *geoms, std::atomic *geompos_in, bool com } } -long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { +long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { double merge_fraction = 1; double mingap_fraction = 1; double minextent_fraction = 1; @@ -1585,6 +1585,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } } + size_t detail_reduced = 0; bool first_time = true; // This only loops if the tile data didn't fit, in which case the detail // goes down and the progress indicator goes backward for the next try. @@ -1732,6 +1733,9 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch bool dropping_by_rate = false; // are we dropping anything by rate in this tile, or keeping it only as part of a multiplier? unsigned long long next_feature_previndex = 0; + strategy strategy; + strategy.detail_reduced = detail_reduced; + for (size_t seq = 0;; seq++) { serial_feature sf; ssize_t which_serial_feature = -1; @@ -1800,7 +1804,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) { preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); - strategy->dropped_by_rate++; + strategy.dropped_by_rate++; can_stop_early = false; continue; } @@ -1815,7 +1819,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (gamma > 0) { if (manage_gap(sf.index, &previndex, scale, gamma, &gap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) { preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); - strategy->dropped_by_gamma++; + strategy.dropped_by_gamma++; drop_rest = true; can_stop_early = false; continue; @@ -1843,7 +1847,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); - strategy->coalesced_as_needed++; + strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; continue; @@ -1875,7 +1879,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); - strategy->coalesced_as_needed++; + strategy.coalesced_as_needed++; drop_rest = true; continue; } @@ -1886,7 +1890,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch features[which_serial_feature].coalesced = true; coalesced_area += sf.extent; preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); - strategy->coalesced_as_needed++; + strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; continue; @@ -1908,7 +1912,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch features[which_serial_feature].coalesced = true; coalesced_area += sf.extent; preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); - strategy->coalesced_as_needed++; + strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; continue; @@ -1929,7 +1933,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch coalesce_geometry(features[which_serial_feature], sf); features[which_serial_feature].coalesced = true; preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); - strategy->coalesced_as_needed++; + strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; continue; @@ -1958,7 +1962,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (!prevent_tiny && !additional[A_GRID_LOW_ZOOMS]) { sf.geometry = reduce_tiny_poly(sf.geometry, z, line_detail, &still_need_simplification_after_reduction, &simplified_away_by_reduction, &accum_area, tiny_polygon_size); if (simplified_away_by_reduction) { - strategy->tiny_polygons++; + strategy.tiny_polygons++; } if (sf.geometry.size() == 0) { continue; @@ -2652,7 +2656,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch continue; } } else { - strategy->detail_reduced++; + detail_reduced++; } } else { if (pthread_mutex_lock(&db_lock) != 0) { @@ -2678,9 +2682,11 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch skip_children_out.insert(zxy(z, tx, ty)); } + strategy_out->add_from(strategy); return count; } } else { + strategy_out->add_from(strategy); return count; } } @@ -3180,3 +3186,12 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: } return maxzoom; } + +void atomic_strategy::add_from(struct strategy const &src) { + dropped_by_rate += src.dropped_by_rate; + dropped_by_gamma += src.dropped_by_gamma; + dropped_as_needed += src.dropped_as_needed; + coalesced_as_needed += src.coalesced_as_needed; + detail_reduced += src.detail_reduced; + tiny_polygons += src.tiny_polygons; +} diff --git a/tile.hpp b/tile.hpp index f679f85d..4401d10a 100644 --- a/tile.hpp +++ b/tile.hpp @@ -27,6 +27,8 @@ struct atomic_strategy { detail_reduced(0), tiny_polygons(0) { } + + void add_from(struct strategy const &src); }; struct strategy { @@ -35,9 +37,10 @@ struct strategy { size_t dropped_as_needed = 0; size_t coalesced_as_needed = 0; size_t detail_reduced = 0; + size_t tiny_polygons = 0; + size_t tile_size = 0; size_t feature_count = 0; - size_t tiny_polygons = 0; strategy(const atomic_strategy &s, size_t ts, size_t fc) { dropped_by_rate = s.dropped_by_rate;