Compare commits

...
6 Commits
8 changed files with 1149 additions and 245 deletions
+5
View File
@@ -1,3 +1,8 @@
# 2.78.0
* Fix potential infinite loops in as-needed dropping and coalescing. When the threshold cannot be increased,
it is now an error, rather than falling back to trying to lower the detail.
# 2.77.0
* Add --deduplicate-by-id option to tippecanoe-overzoom
+23
View File
@@ -2469,3 +2469,26 @@ drawvec fix_polygon(const drawvec &geom, bool use_winding, bool reverse_winding)
return out;
}
bool line_is_too_small(drawvec const &geometry, int z, int detail) {
if (geometry.size() == 0) {
return true;
}
long long x = 0, y = 0;
for (auto &g : geometry) {
if (g.op == VT_MOVETO) {
x = std::llround((double) g.x / (1LL << (32 - detail - z)));
y = std::llround((double) g.y / (1LL << (32 - detail - z)));
} else {
long long xx = std::llround((double) g.x / (1LL << (32 - detail - z)));
long long yy = std::llround((double) g.y / (1LL << (32 - detail - z)));
if (xx != x || yy != y) {
return false;
}
}
}
return true;
}
+1
View File
@@ -172,5 +172,6 @@ void get_quadkey_bounds(long long xmin, long long ymin, long long xmax, long lon
unsigned long long *start, unsigned long long *end);
clipbbox parse_clip_poly(std::string arg);
bool line_is_too_small(drawvec const &geometry, int z, int detail);
#endif
+73 -43
View File
@@ -676,7 +676,9 @@ static void *simplification_worker(void *v) {
for (size_t i = a->task; i < (*features).size(); i += a->tasks) {
double area = 0;
if (!a->trying_to_stop_early) {
if (a->trying_to_stop_early) {
area = get_mp_area((*features)[i]->geometry);
} else {
area = simplify_feature(&*((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos, *(a->shared_nodes_bloom));
}
@@ -701,8 +703,13 @@ static void *simplification_worker(void *v) {
}
if (geom.size() < 3) {
if (area > 0) {
if (area > 0 && !additional[A_GRID_LOW_ZOOMS]) {
// area is in world coordinates, calculated before scaling down
//
// Don't revive if we are gridding low zooms, because
// the expectation there is that only the features that
// happen to span integer divisions of the tile grid
// will survive
geom = revive_polygon(before, area, z, out_detail);
} else {
geom.clear();
@@ -712,15 +719,19 @@ static void *simplification_worker(void *v) {
}
}
if (t == VT_POLYGON && additional[A_GENERATE_POLYGON_LABEL_POINTS]) {
t = (*features)[i]->t = VT_POINT;
geom = checkerboard_anchors(from_tile_scale(geom, z, out_detail), (*features)[i]->tx, (*features)[i]->ty, z, (*features)[i]->label_point);
to_tile_scale(geom, z, out_detail);
}
if ((*features)[i]->index == 0) {
(*features)[i]->index = i;
}
#if 0
if (geom.size() == 0 && !additional[A_GRID_LOW_ZOOMS]) {
fprintf(stderr, "simplified away to nothing\n");
for (auto const &g : geom) {
fprintf(stderr, "(%d)%lld,%lld ", g.op, g.x, g.y);
}
fprintf(stderr, "\n");
exit(EXIT_FAILURE);
}
#endif
(*features)[i]->geometry = std::move(geom);
}
@@ -769,7 +780,7 @@ static unsigned long long choose_mingap(std::vector<unsigned long long> &gaps, d
std::stable_sort(gaps.begin(), gaps.end());
size_t ix = (gaps.size() - 1) * (1 - f);
while (ix + 1 < gaps.size() && gaps[ix] == existing_gap) {
while (ix + 1 < gaps.size() && gaps[ix] <= existing_gap) {
ix++;
}
@@ -809,7 +820,7 @@ static long long choose_minextent(std::vector<long long> &extents, double f, lon
std::stable_sort(extents.begin(), extents.end());
size_t ix = (extents.size() - 1) * (1 - f);
while (ix + 1 < extents.size() && extents[ix] == existing_extent) {
while (ix + 1 < extents.size() && extents[ix] <= existing_extent) {
ix++;
}
@@ -824,7 +835,7 @@ static unsigned long long choose_mindrop_sequence(std::vector<unsigned long long
std::stable_sort(drop_sequences.begin(), drop_sequences.end());
size_t ix = (drop_sequences.size() - 1) * (1 - f);
while (ix + 1 < drop_sequences.size() && drop_sequences[ix] == existing_drop_sequence) {
while (ix + 1 < drop_sequences.size() && drop_sequences[ix] <= existing_drop_sequence) {
ix++;
}
@@ -1551,26 +1562,6 @@ bool find_feature_to_accumulate_onto(std::vector<std::shared_ptr<serial_feature>
return false;
}
static bool line_is_too_small(drawvec const &geometry, int z, int detail) {
if (geometry.size() == 0) {
return true;
}
long long x = std::round((double) geometry[0].x / (1LL << (32 - detail - z)));
long long y = std::round((double) geometry[0].y / (1LL << (32 - detail - z)));
for (auto &g : geometry) {
long long xx = std::round((double) g.x / (1LL << (32 - detail - z)));
long long yy = std::round((double) g.y / (1LL << (32 - detail - z)));
if (xx != x || yy != y) {
return false;
}
}
return true;
}
// Keep only a sample of 100K extents for feature dropping,
// to avoid spending lots of memory on a complete list when there are
// hundreds of millions of features.
@@ -2090,6 +2081,25 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (line_is_too_small(sf.geometry, z, line_detail)) {
continue;
}
if (sf.t == VT_POLYGON && get_mp_area(sf.geometry) <= 0) {
continue;
}
}
// Make label anchors early in tiling, even though it requires simplifying early,
// so that if there is no label anchor for this feature in this tile,
// we find out now rather than after we have already decided that there
// are too_many_bytes.
if (sf.t == VT_POLYGON && additional[A_GENERATE_POLYGON_LABEL_POINTS]) {
drawvec ngeom = simplify_lines(sf.geometry, z, tx, ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), sf.simplification, sf.t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, "");
if (ngeom.size() == 0) {
continue;
}
sf.geometry = checkerboard_anchors(ngeom, tx, ty, z, sf.label_point);
if (sf.geometry.size() == 0) {
continue;
}
sf.t = VT_POINT;
}
unsigned long long sfindex = sf.index;
@@ -2194,6 +2204,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// have been assembled globally, although that also means that simplification
// may not be very effective for reducing memory usage.
fprintf(stderr, "simplifying through\n");
for (; simplified_geometry_through < features.size(); simplified_geometry_through++) {
simplify_feature(&*features[simplified_geometry_through], shared_nodes, shared_nodes_map, nodepos, shared_nodes_bloom);
@@ -2654,9 +2665,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
line_detail++; // to keep it the same when the loop decrements it
continue;
} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
mingap_fraction = mingap_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.80;
mingap_fraction = std::min(1.0, mingap_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.80);
unsigned long long m = choose_mingap(gaps, mingap_fraction, mingap);
if (m != mingap) {
if (m > mingap) {
mingap = m;
if (mingap > arg->mingap_out) {
arg->mingap_out = mingap;
@@ -2667,11 +2678,14 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
line_detail++;
continue;
} else {
fprintf(stderr, "Can't increase feature gap threshold further\n");
exit(EXIT_INCOMPLETE);
}
} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
minextent_fraction = minextent_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.75;
minextent_fraction = std::min(1.0, minextent_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.75);
long long m = choose_minextent(extents, minextent_fraction, minextent);
if (m != minextent) {
if (m > minextent) {
minextent = m;
if (minextent > arg->minextent_out) {
arg->minextent_out = minextent;
@@ -2682,14 +2696,17 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
line_detail++;
continue;
} else {
fprintf(stderr, "Can't increase feature extent threshold further\n");
exit(EXIT_INCOMPLETE);
}
} else if (feature_count > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
// The 95% is a guess to avoid too many retries
// and probably actually varies based on how much duplicated metadata there is
mindrop_sequence_fraction = mindrop_sequence_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.95;
mindrop_sequence_fraction = std::min(1.0, mindrop_sequence_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.95);
unsigned long long m = choose_mindrop_sequence(drop_sequences, mindrop_sequence_fraction, mindrop_sequence);
if (m != mindrop_sequence) {
if (m > mindrop_sequence) {
mindrop_sequence = m;
if (mindrop_sequence > arg->mindrop_sequence_out) {
if (!prevent[P_DYNAMIC_DROP]) {
@@ -2702,6 +2719,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
line_detail++; // to keep it the same when the loop decrements it
continue;
} else {
fprintf(stderr, "Can't increase feature fraction threshold further\n");
exit(EXIT_INCOMPLETE);
}
} else {
fprintf(stderr, "Try using --drop-fraction-as-needed or --drop-densest-as-needed.\n");
@@ -2763,10 +2783,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
fprintf(stderr, "Going to try gamma of %0.3f to make it fit\n", gamma);
}
line_detail++; // to keep it the same when the loop decrements it
continue;
} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
mingap_fraction = mingap_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.80;
mingap_fraction = std::min(1.0, mingap_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.80);
unsigned long long m = choose_mingap(gaps, mingap_fraction, mingap);
if (m != mingap) {
if (m > mingap) {
mingap = m;
if (mingap > arg->mingap_out) {
arg->mingap_out = mingap;
@@ -2777,11 +2798,14 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
line_detail++;
continue;
} else {
fprintf(stderr, "Can't increase feature gap threshold further\n");
exit(EXIT_INCOMPLETE);
}
} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
minextent_fraction = minextent_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75;
minextent_fraction = std::min(1.0, minextent_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75);
long long m = choose_minextent(extents, minextent_fraction, minextent);
if (m != minextent) {
if (m > minextent) {
minextent = m;
if (minextent > arg->minextent_out) {
arg->minextent_out = minextent;
@@ -2792,11 +2816,14 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
line_detail++;
continue;
} else {
fprintf(stderr, "Can't increase feature extent threshold further\n");
exit(EXIT_INCOMPLETE);
}
} else if (feature_count > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
mindrop_sequence_fraction = mindrop_sequence_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75;
mindrop_sequence_fraction = std::min(1.0, mindrop_sequence_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75);
unsigned long long m = choose_mindrop_sequence(drop_sequences, mindrop_sequence_fraction, mindrop_sequence);
if (m != mindrop_sequence) {
if (m > mindrop_sequence) {
mindrop_sequence = m;
if (mindrop_sequence > arg->mindrop_sequence_out) {
if (!prevent[P_DYNAMIC_DROP]) {
@@ -2809,6 +2836,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
line_detail++;
continue;
} else {
fprintf(stderr, "Can't increase feature fraction threshold further\n");
exit(EXIT_INCOMPLETE);
}
} else {
detail_reduced++;
+9
View File
@@ -152,3 +152,12 @@ TEST_CASE("mvt_geometry bbox") {
REQUIRE(start == 0x1c84fc0000000000);
REQUIRE(end == 0x1c84ffffffffffff);
}
TEST_CASE("line_is_too_small") {
drawvec dv;
dv.emplace_back(VT_MOVETO, 4243099709, 2683872952);
dv.emplace_back(VT_LINETO, 4243102487, 2683873977);
dv.emplace_back(VT_MOVETO, -51867587, 2683872952);
dv.emplace_back(VT_LINETO, -51864809, 2683873977);
REQUIRE(line_is_too_small(dv, 0, 10));
}
+1 -1
View File
@@ -1,6 +1,6 @@
#ifndef VERSION_HPP
#define VERSION_HPP
#define VERSION "v2.77.0"
#define VERSION "v2.78.0"
#endif