Add --deduplicate-by-id option to tippecanoe-overzoom (#331)

* Add option to deduplicate by feature ID in overzoom

* Add test, fix default

* Update version and changelog

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.77.0
+
+* Add --deduplicate-by-id option to tippecanoe-overzoom
+
 # 2.76.0
 
 * Add missing case for accumulating the mean of attributes that are inconsistently present

Makefile

@@ -443,6 +443,16 @@ overzoom-test: tippecanoe-overzoom
 	./tippecanoe-overzoom -o tests/pbf/squirrels-13-2413-3077-clip.pbf --clip-polygon-file tests/pbf/squirrels-clip.json tests/pbf/squirrels-13-2413-3077.pbf 13/2413/3077 13/2413/3077
 	cmp tests/pbf/squirrels-13-2413-3077-clip.pbf /dev/null  # clipped away
 	rm tests/pbf/squirrels-13-2413-3077-clip.pbf
+	# Deduplication by feature ID
+	./tippecanoe -z0 -f -e tests/pbf/1.json.dir -l layer tests/pbf/1.json
+	./tippecanoe -z0 -f -e tests/pbf/2.json.dir -l layer tests/pbf/2.json
+	./tippecanoe-overzoom -o tests/pbf/merged-nodedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0
+	./tippecanoe-decode tests/pbf/merged-nodedup.pbf 0 0 0 > tests/pbf/merged-nodedup.pbf.json.check
+	cmp tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-nodedup.pbf.json
+	./tippecanoe-overzoom --deduplicate-by-id -o tests/pbf/merged-dedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0
+	./tippecanoe-decode tests/pbf/merged-dedup.pbf 0 0 0 > tests/pbf/merged-dedup.pbf.json.check
+	cmp tests/pbf/merged-dedup.pbf.json.check tests/pbf/merged-dedup.pbf.json
+	rm -r tests/pbf/1.json.dir tests/pbf/2.json.dir tests/pbf/merged-nodedup.pbf tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-dedup.pbf.json.check
 
 join-test: tippecanoe tippecanoe-decode tile-join
 	./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}'

clip.cpp

@@ -1269,7 +1269,8 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
 		     std::string const &accumulate_numeric, size_t feature_limit,
-		     std::vector<clipbbox> const &clipbboxes) {
+		     std::vector<clipbbox> const &clipbboxes,
+		     bool deduplicate_by_id) {
 	std::vector<source_tile> decoded;
 
 	for (auto const &t : tiles) {
@@ -1295,7 +1296,7 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
 		decoded.push_back(out);
 	}
 
-	return overzoom(decoded, nz, nx, ny, detail_or_unspecified, buffer, keep, exclude, exclude_prefix, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, feature_limit, clipbboxes);
+	return overzoom(decoded, nz, nx, ny, detail_or_unspecified, buffer, keep, exclude, exclude_prefix, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, feature_limit, clipbboxes, deduplicate_by_id);
 }
 
 // like a minimal serial_feature, but with mvt_feature-style attributes
@@ -1499,7 +1500,8 @@ static bool feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
 			std::unordered_map<std::string, attribute_op> const &attribute_accum,
 			std::string const &accumulate_numeric,
 			key_pool &key_pool, int buffer, bool include_nonaggregate,
-			std::vector<clipbbox> const &clipbboxes, int nz, int nx, int ny) {
+			std::vector<clipbbox> const &clipbboxes, int nz, int nx, int ny,
+			std::set<unsigned long long> *deduplicate_ids) {
 	// Add geometry to output feature
 
 	drawvec geom = features[0].geom;
@@ -1582,6 +1584,14 @@ static bool feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
 
 	// ID and attributes, if it didn't get clipped away
 
+	if (features[0].has_id && deduplicate_ids != NULL) {
+		if (deduplicate_ids->find(features[0].id) != deduplicate_ids->end()) {
+			outfeature.geometry.clear();
+		} else {
+			deduplicate_ids->insert(features[0].id);
+		}
+	}
+
 	if (outfeature.geometry.size() > 0) {
 		if (features[0].has_id) {
 			outfeature.has_id = true;
@@ -1973,7 +1983,7 @@ mvt_tile assign_to_bins(mvt_tile &features,
 			if (feature_out(outfeatures[i], outlayer,
 					keep, exclude, exclude_prefix, attribute_accum,
 					accumulate_numeric, key_pool, buffer, true,
-					clipbboxes, z, x, y)) {
+					clipbboxes, z, x, y, NULL)) {
 				mvt_feature &nfeature = outlayer.features.back();
 				mvt_value val;
 				val.type = mvt_uint;
@@ -2010,10 +2020,14 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
 		     std::string const &accumulate_numeric, size_t feature_limit,
-		     std::vector<clipbbox> const &clipbboxes) {
+		     std::vector<clipbbox> const &clipbboxes,
+		     bool deduplicate_by_id) {
 	mvt_tile outtile;
 	key_pool key_pool;
 
+	// map from layer name to ids used in that layer
+	std::map<std::string, std::set<unsigned long long>> deduplicate_ids;
+
 	for (auto const &tile : tiles) {
 		for (auto const &layer : tile.tile.layers) {
 			mvt_layer *outlayer = NULL;
@@ -2040,6 +2054,16 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 				outlayer = &outtile.layers.back();
 			}
 
+			std::set<unsigned long long> *deduplicate_by_id_set = NULL;
+			if (deduplicate_by_id) {
+				auto layer_deduplicate_ids = deduplicate_ids.find(layer.name);
+				if (layer_deduplicate_ids == deduplicate_ids.end()) {
+					deduplicate_ids.emplace(layer.name, std::set<unsigned long long>());
+					layer_deduplicate_ids = deduplicate_ids.find(layer.name);
+				}
+				deduplicate_by_id_set = &layer_deduplicate_ids->second;
+			}
+
 			std::vector<tile_feature> pending_tile_features;
 			double accum_area = 0;
 
@@ -2161,7 +2185,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 
 				if (flush_multiplier_cluster) {
 					if (pending_tile_features.size() > 0) {
-						feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector<clipbbox>(), nz, nx, ny);
+						feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector<clipbbox>(), nz, nx, ny, deduplicate_by_id_set);
 						if (outlayer->features.size() >= feature_limit) {
 							break;
 						}
@@ -2221,7 +2245,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 			}
 
 			if (pending_tile_features.size() > 0) {
-				feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector<clipbbox>(), nz, nx, ny);
+				feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector<clipbbox>(), nz, nx, ny, deduplicate_by_id_set);
 				pending_tile_features.clear();
 				if (outlayer->features.size() >= feature_limit) {
 					break;
@@ -2261,7 +2285,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 								     detail_or_unspecified, buffer, keep, exclude, exclude_prefix, false, NULL,
 								     demultiply, filter, preserve_input_order, attribute_accum, unidecode_data,
 								     simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric,
-								     1, clipbboxes);
+								     1, clipbboxes, deduplicate_by_id);
 					if (child.size() > 0) {
 						next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y);
 					}

geometry.hpp

@@ -147,7 +147,8 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
 		     std::string const &accumulate_numeric, size_t feature_limit,
-		     std::vector<clipbbox> const &clipbboxes);
+		     std::vector<clipbbox> const &clipbboxes,
+		     bool deduplicate_by_id);
 
 std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int ny,
 		     int detail, int buffer,
@@ -162,7 +163,8 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
 		     std::string const &accumulate_numeric, size_t feature_limit,
-		     std::vector<clipbbox> const &clipbboxes);
+		     std::vector<clipbbox> const &clipbboxes,
+		     bool deduplicate_by_id);
 
 draw center_of_mass_mp(const drawvec &dv);
 

overzoom.cpp

@@ -19,6 +19,7 @@ int detail = 12;  // tippecanoe-style: mvt extent == 1 << detail
 int buffer = 5;	  // tippecanoe-style: mvt buffer == extent * buffer / 256;
 bool demultiply = false;
 bool do_compress = true;
+bool deduplicate_by_id = false;
 
 std::string filter;
 bool preserve_input_order = false;
@@ -94,6 +95,7 @@ int main(int argc, char **argv) {
 		{"clip-bounding-box", required_argument, 0, 'k' & 0x1F},
 		{"clip-polygon", required_argument, 0, 'l' & 0x1F},
 		{"clip-polygon-file", required_argument, 0, 'm' & 0x1F},
+		{"deduplicate-by-id", no_argument, 0, 'i' & 0x1F},
 
 		{0, 0, 0, 0},
 	};
@@ -213,6 +215,11 @@ int main(int argc, char **argv) {
 			break;
 		}
 
+		case 'i' & 0x1F: {
+			deduplicate_by_id = true;
+			break;
+		}
+
 		default:
 			fprintf(stderr, "Unrecognized flag -%c\n", i);
 			usage(argv);
@@ -354,7 +361,7 @@ int main(int argc, char **argv) {
 			its.push_back(std::move(t));
 		}
 
-		out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes);
+		out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes, deduplicate_by_id);
 	}
 
 	FILE *f = fopen(outfile, "wb");

tests/pbf/1.json

@@ -0,0 +1,3 @@
+{"type":"Feature","properties":{"what":"no id"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12345,"properties":{"what":"will survive"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12346,"properties":{"what":"will win over the duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}}

tests/pbf/2.json

@@ -0,0 +1,3 @@
+{"type":"Feature","properties":{"what":"no id again"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12346,"properties":{"what":"will be lost as a duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12347,"properties":{"what":"will be added"},"geometry":{"type":"Point","coordinates":[0,0]}}

tests/pbf/merged-dedup.pbf.json

@@ -0,0 +1,13 @@
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+] }

tests/pbf/merged-nodedup.pbf.json

@@ -0,0 +1,15 @@
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12346, "properties": { "what": "will be lost as a duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+] }

tile-join.cpp

@@ -982,7 +982,7 @@ struct tileset_reader {
 						   false, &next_overzoomed_tiles, false, NULL, false,
 						   std::unordered_map<std::string, attribute_op>(), unidecode_data, 0, 0,
 						   std::vector<mvt_layer>(), "", "", SIZE_MAX,
-						   std::vector<clipbbox>());
+						   std::vector<clipbbox>(), false);
 			return ret;
 		}
 

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.76.0"
+#define VERSION "v2.77.0"
 
 #endif