From 68ab8dcc229f95b8b25877697d5e8d66783af503 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Thu, 24 Jul 2025 13:21:10 -0700 Subject: [PATCH] Deduplicate in tippecanoe-overzoom even when the duplicate is clipped away (#353) * Deduplicate by ID even when the duplicate is clipped away * Test that deduplication works across tile boundaries * Update version and changelog --- CHANGELOG.md | 5 +++++ Makefile | 8 ++++---- clip.cpp | 18 ++++++++++++++++++ tests/pbf/1.json | 6 +++--- tests/pbf/2.json | 6 +++--- tests/pbf/merged-dedup.pbf.json | 12 +++++------- tests/pbf/merged-nodedup.pbf.json | 14 ++++++-------- version.hpp | 2 +- 8 files changed, 45 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62c074a6..65fcfa90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# 2.79.0 + +* When deduplicating features by ID in tippecanoe-overzoom, be careful + to track even features that have been clipped away. + # 2.78.0 * Fix potential infinite loops in as-needed dropping and coalescing. diff --git a/Makefile b/Makefile index fefc84f5..f0598ea9 100644 --- a/Makefile +++ b/Makefile @@ -446,11 +446,11 @@ overzoom-test: tippecanoe-overzoom # Deduplication by feature ID ./tippecanoe -z0 -f -e tests/pbf/1.json.dir -l layer tests/pbf/1.json ./tippecanoe -z0 -f -e tests/pbf/2.json.dir -l layer tests/pbf/2.json - ./tippecanoe-overzoom -o tests/pbf/merged-nodedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0 - ./tippecanoe-decode tests/pbf/merged-nodedup.pbf 0 0 0 > tests/pbf/merged-nodedup.pbf.json.check + ./tippecanoe-overzoom -b0 -o tests/pbf/merged-nodedup.pbf -t 1/1/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0 + ./tippecanoe-decode tests/pbf/merged-nodedup.pbf 1 1 0 > tests/pbf/merged-nodedup.pbf.json.check cmp tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-nodedup.pbf.json - ./tippecanoe-overzoom --deduplicate-by-id -o tests/pbf/merged-dedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0 - ./tippecanoe-decode tests/pbf/merged-dedup.pbf 0 0 0 > tests/pbf/merged-dedup.pbf.json.check + ./tippecanoe-overzoom -b0 --deduplicate-by-id -o tests/pbf/merged-dedup.pbf -t 1/1/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0 + ./tippecanoe-decode tests/pbf/merged-dedup.pbf 1 1 0 > tests/pbf/merged-dedup.pbf.json.check cmp tests/pbf/merged-dedup.pbf.json.check tests/pbf/merged-dedup.pbf.json rm -r tests/pbf/1.json.dir tests/pbf/2.json.dir tests/pbf/merged-nodedup.pbf tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-dedup.pbf.json.check diff --git a/clip.cpp b/clip.cpp index ca0af8e3..cdaa07c5 100644 --- a/clip.cpp +++ b/clip.cpp @@ -2145,6 +2145,9 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int long long b = outtilesize * buffer / 256; if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) { // quick exclusion by bounding box + if (deduplicate_by_id && feature.has_id) { + deduplicate_by_id_set->insert(feature.id); + } continue; } @@ -2160,6 +2163,9 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int if (geom.size() == 0) { // clipped away + if (deduplicate_by_id && feature.has_id) { + deduplicate_by_id_set->insert(feature.id); + } continue; } @@ -2194,6 +2200,10 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int std::set exclude_attributes; if (filter != NULL && !evaluate(feature, layer, filter, exclude_attributes, nz, unidecode_data)) { + // filtered away + if (deduplicate_by_id && feature.has_id) { + deduplicate_by_id_set->insert(feature.id); + } continue; } @@ -2231,6 +2241,14 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int geom = close_poly(geom); } + if (geom.size() == 0) { + // simplified away + if (deduplicate_by_id && feature.has_id) { + deduplicate_by_id_set->insert(feature.id); + } + continue; + } + tile_feature tf; tf.geom = std::move(geom); tf.t = t; diff --git a/tests/pbf/1.json b/tests/pbf/1.json index 14e2e105..2de7407d 100644 --- a/tests/pbf/1.json +++ b/tests/pbf/1.json @@ -1,3 +1,3 @@ -{"type":"Feature","properties":{"what":"no id"},"geometry":{"type":"Point","coordinates":[0,0]}} -{"type":"Feature","id":12345,"properties":{"what":"will survive"},"geometry":{"type":"Point","coordinates":[0,0]}} -{"type":"Feature","id":12346,"properties":{"what":"will win over the duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}} +{"type":"Feature","properties":{"what":"no id"},"geometry":{"type":"Point","coordinates":[1,1]}} +{"type":"Feature","id":12345,"properties":{"what":"will survive"},"geometry":{"type":"Point","coordinates":[1,1]}} +{"type":"Feature","id":12346,"properties":{"what":"will win over the duplicate even though it's in another tile"},"geometry":{"type":"Point","coordinates":[-1,-1]}} diff --git a/tests/pbf/2.json b/tests/pbf/2.json index 9506f22f..b0b22034 100644 --- a/tests/pbf/2.json +++ b/tests/pbf/2.json @@ -1,3 +1,3 @@ -{"type":"Feature","properties":{"what":"no id again"},"geometry":{"type":"Point","coordinates":[0,0]}} -{"type":"Feature","id":12346,"properties":{"what":"will be lost as a duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}} -{"type":"Feature","id":12347,"properties":{"what":"will be added"},"geometry":{"type":"Point","coordinates":[0,0]}} +{"type":"Feature","properties":{"what":"no id again"},"geometry":{"type":"Point","coordinates":[1,1]}} +{"type":"Feature","id":12346,"properties":{"what":"will be lost as a duplicate"},"geometry":{"type":"Point","coordinates":[1,1]}} +{"type":"Feature","id":12347,"properties":{"what":"will be added"},"geometry":{"type":"Point","coordinates":[1,1]}} diff --git a/tests/pbf/merged-dedup.pbf.json b/tests/pbf/merged-dedup.pbf.json index 5727adba..10c9fbd0 100644 --- a/tests/pbf/merged-dedup.pbf.json +++ b/tests/pbf/merged-dedup.pbf.json @@ -1,13 +1,11 @@ -{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 0 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } } , -{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } } , -{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } } , -{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } -, -{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } } ] } ] } diff --git a/tests/pbf/merged-nodedup.pbf.json b/tests/pbf/merged-nodedup.pbf.json index dfd65ab2..343150a8 100644 --- a/tests/pbf/merged-nodedup.pbf.json +++ b/tests/pbf/merged-nodedup.pbf.json @@ -1,15 +1,13 @@ -{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 0 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } } , -{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } } , -{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } } , -{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +{ "type": "Feature", "id": 12346, "properties": { "what": "will be lost as a duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } } , -{ "type": "Feature", "id": 12346, "properties": { "what": "will be lost as a duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } -, -{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } } ] } ] } diff --git a/version.hpp b/version.hpp index b9593e1d..c8a13cad 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v2.78.0" +#define VERSION "v2.79.0" #endif