Deduplicate in tippecanoe-overzoom even when the duplicate is clipped away (#353)

* Deduplicate by ID even when the duplicate is clipped away

* Test that deduplication works across tile boundaries

* Update version and changelog

CHANGELOG.md

@@ -1,3 +1,8 @@
+# 2.79.0
+
+* When deduplicating features by ID in tippecanoe-overzoom, be careful
+  to track even features that have been clipped away.
+
 # 2.78.0
 
 * Fix potential infinite loops in as-needed dropping and coalescing.

Makefile

@@ -446,11 +446,11 @@ overzoom-test: tippecanoe-overzoom
 	# Deduplication by feature ID
 	./tippecanoe -z0 -f -e tests/pbf/1.json.dir -l layer tests/pbf/1.json
 	./tippecanoe -z0 -f -e tests/pbf/2.json.dir -l layer tests/pbf/2.json
-	./tippecanoe-overzoom -o tests/pbf/merged-nodedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0
+	./tippecanoe-overzoom -b0 -o tests/pbf/merged-nodedup.pbf -t 1/1/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0
-	./tippecanoe-decode tests/pbf/merged-nodedup.pbf 0 0 0 > tests/pbf/merged-nodedup.pbf.json.check
+	./tippecanoe-decode tests/pbf/merged-nodedup.pbf 1 1 0 > tests/pbf/merged-nodedup.pbf.json.check
 	cmp tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-nodedup.pbf.json
-	./tippecanoe-overzoom --deduplicate-by-id -o tests/pbf/merged-dedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0
+	./tippecanoe-overzoom -b0 --deduplicate-by-id -o tests/pbf/merged-dedup.pbf -t 1/1/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0
-	./tippecanoe-decode tests/pbf/merged-dedup.pbf 0 0 0 > tests/pbf/merged-dedup.pbf.json.check
+	./tippecanoe-decode tests/pbf/merged-dedup.pbf 1 1 0 > tests/pbf/merged-dedup.pbf.json.check
 	cmp tests/pbf/merged-dedup.pbf.json.check tests/pbf/merged-dedup.pbf.json
 	rm -r tests/pbf/1.json.dir tests/pbf/2.json.dir tests/pbf/merged-nodedup.pbf tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-dedup.pbf.json.check
 

clip.cpp

@@ -2145,6 +2145,9 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 					long long b = outtilesize * buffer / 256;
 					if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) {
 						// quick exclusion by bounding box
+						if (deduplicate_by_id && feature.has_id) {
+							deduplicate_by_id_set->insert(feature.id);
+						}
 						continue;
 					}
 
@@ -2160,6 +2163,9 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 
 				if (geom.size() == 0) {
 					// clipped away
+					if (deduplicate_by_id && feature.has_id) {
+						deduplicate_by_id_set->insert(feature.id);
+					}
 					continue;
 				}
 
@@ -2194,6 +2200,10 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 
 				std::set<std::string> exclude_attributes;
 				if (filter != NULL && !evaluate(feature, layer, filter, exclude_attributes, nz, unidecode_data)) {
+					// filtered away
+					if (deduplicate_by_id && feature.has_id) {
+						deduplicate_by_id_set->insert(feature.id);
+					}
 					continue;
 				}
 
@@ -2231,6 +2241,14 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 					geom = close_poly(geom);
 				}
 
+				if (geom.size() == 0) {
+					// simplified away
+					if (deduplicate_by_id && feature.has_id) {
+						deduplicate_by_id_set->insert(feature.id);
+					}
+					continue;
+				}
+
 				tile_feature tf;
 				tf.geom = std::move(geom);
 				tf.t = t;

tests/pbf/1.json

@@ -1,3 +1,3 @@
-{"type":"Feature","properties":{"what":"no id"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","properties":{"what":"no id"},"geometry":{"type":"Point","coordinates":[1,1]}}
-{"type":"Feature","id":12345,"properties":{"what":"will survive"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12345,"properties":{"what":"will survive"},"geometry":{"type":"Point","coordinates":[1,1]}}
-{"type":"Feature","id":12346,"properties":{"what":"will win over the duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12346,"properties":{"what":"will win over the duplicate even though it's in another tile"},"geometry":{"type":"Point","coordinates":[-1,-1]}}

tests/pbf/2.json

@@ -1,3 +1,3 @@
-{"type":"Feature","properties":{"what":"no id again"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","properties":{"what":"no id again"},"geometry":{"type":"Point","coordinates":[1,1]}}
-{"type":"Feature","id":12346,"properties":{"what":"will be lost as a duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12346,"properties":{"what":"will be lost as a duplicate"},"geometry":{"type":"Point","coordinates":[1,1]}}
-{"type":"Feature","id":12347,"properties":{"what":"will be added"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12347,"properties":{"what":"will be added"},"geometry":{"type":"Point","coordinates":[1,1]}}

tests/pbf/merged-dedup.pbf.json

@@ -1,13 +1,11 @@
-{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 0 }, "features": [
 { "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [
-{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
 ,
-{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
 ,
-{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
 ,
-{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
-,
-{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
 ] }
 ] }

tests/pbf/merged-nodedup.pbf.json

@@ -1,15 +1,13 @@
-{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 0 }, "features": [
 { "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [
-{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
 ,
-{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
 ,
-{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
 ,
-{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+{ "type": "Feature", "id": 12346, "properties": { "what": "will be lost as a duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
 ,
-{ "type": "Feature", "id": 12346, "properties": { "what": "will be lost as a duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
-,
-{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
 ] }
 ] }

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.78.0"
+#define VERSION "v2.79.0"
 
 #endif