diff --git a/CHANGELOG.md b/CHANGELOG.md index 0915f7e2..7737171d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 2.77.0 + +* Add --deduplicate-by-id option to tippecanoe-overzoom + # 2.76.0 * Add missing case for accumulating the mean of attributes that are inconsistently present diff --git a/Makefile b/Makefile index fc40467b..fefc84f5 100644 --- a/Makefile +++ b/Makefile @@ -443,6 +443,16 @@ overzoom-test: tippecanoe-overzoom ./tippecanoe-overzoom -o tests/pbf/squirrels-13-2413-3077-clip.pbf --clip-polygon-file tests/pbf/squirrels-clip.json tests/pbf/squirrels-13-2413-3077.pbf 13/2413/3077 13/2413/3077 cmp tests/pbf/squirrels-13-2413-3077-clip.pbf /dev/null # clipped away rm tests/pbf/squirrels-13-2413-3077-clip.pbf + # Deduplication by feature ID + ./tippecanoe -z0 -f -e tests/pbf/1.json.dir -l layer tests/pbf/1.json + ./tippecanoe -z0 -f -e tests/pbf/2.json.dir -l layer tests/pbf/2.json + ./tippecanoe-overzoom -o tests/pbf/merged-nodedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0 + ./tippecanoe-decode tests/pbf/merged-nodedup.pbf 0 0 0 > tests/pbf/merged-nodedup.pbf.json.check + cmp tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-nodedup.pbf.json + ./tippecanoe-overzoom --deduplicate-by-id -o tests/pbf/merged-dedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0 + ./tippecanoe-decode tests/pbf/merged-dedup.pbf 0 0 0 > tests/pbf/merged-dedup.pbf.json.check + cmp tests/pbf/merged-dedup.pbf.json.check tests/pbf/merged-dedup.pbf.json + rm -r tests/pbf/1.json.dir tests/pbf/2.json.dir tests/pbf/merged-nodedup.pbf tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-dedup.pbf.json.check join-test: tippecanoe tippecanoe-decode tile-join ./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}' diff --git a/clip.cpp b/clip.cpp index 69decd45..d4a6f9ce 100644 --- a/clip.cpp +++ b/clip.cpp @@ -1269,7 +1269,8 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int n double tiny_polygon_size, std::vector const &bins, std::string const &bin_by_id_list, std::string const &accumulate_numeric, size_t feature_limit, - std::vector const &clipbboxes) { + std::vector const &clipbboxes, + bool deduplicate_by_id) { std::vector decoded; for (auto const &t : tiles) { @@ -1295,7 +1296,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int n decoded.push_back(out); } - return overzoom(decoded, nz, nx, ny, detail_or_unspecified, buffer, keep, exclude, exclude_prefix, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, feature_limit, clipbboxes); + return overzoom(decoded, nz, nx, ny, detail_or_unspecified, buffer, keep, exclude, exclude_prefix, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, feature_limit, clipbboxes, deduplicate_by_id); } // like a minimal serial_feature, but with mvt_feature-style attributes @@ -1499,7 +1500,8 @@ static bool feature_out(std::vector const &features, mvt_layer &ou std::unordered_map const &attribute_accum, std::string const &accumulate_numeric, key_pool &key_pool, int buffer, bool include_nonaggregate, - std::vector const &clipbboxes, int nz, int nx, int ny) { + std::vector const &clipbboxes, int nz, int nx, int ny, + std::set *deduplicate_ids) { // Add geometry to output feature drawvec geom = features[0].geom; @@ -1582,6 +1584,14 @@ static bool feature_out(std::vector const &features, mvt_layer &ou // ID and attributes, if it didn't get clipped away + if (features[0].has_id && deduplicate_ids != NULL) { + if (deduplicate_ids->find(features[0].id) != deduplicate_ids->end()) { + outfeature.geometry.clear(); + } else { + deduplicate_ids->insert(features[0].id); + } + } + if (outfeature.geometry.size() > 0) { if (features[0].has_id) { outfeature.has_id = true; @@ -1973,7 +1983,7 @@ mvt_tile assign_to_bins(mvt_tile &features, if (feature_out(outfeatures[i], outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, buffer, true, - clipbboxes, z, x, y)) { + clipbboxes, z, x, y, NULL)) { mvt_feature &nfeature = outlayer.features.back(); mvt_value val; val.type = mvt_uint; @@ -2010,10 +2020,14 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int double tiny_polygon_size, std::vector const &bins, std::string const &bin_by_id_list, std::string const &accumulate_numeric, size_t feature_limit, - std::vector const &clipbboxes) { + std::vector const &clipbboxes, + bool deduplicate_by_id) { mvt_tile outtile; key_pool key_pool; + // map from layer name to ids used in that layer + std::map> deduplicate_ids; + for (auto const &tile : tiles) { for (auto const &layer : tile.tile.layers) { mvt_layer *outlayer = NULL; @@ -2040,6 +2054,16 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int outlayer = &outtile.layers.back(); } + std::set *deduplicate_by_id_set = NULL; + if (deduplicate_by_id) { + auto layer_deduplicate_ids = deduplicate_ids.find(layer.name); + if (layer_deduplicate_ids == deduplicate_ids.end()) { + deduplicate_ids.emplace(layer.name, std::set()); + layer_deduplicate_ids = deduplicate_ids.find(layer.name); + } + deduplicate_by_id_set = &layer_deduplicate_ids->second; + } + std::vector pending_tile_features; double accum_area = 0; @@ -2161,7 +2185,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int if (flush_multiplier_cluster) { if (pending_tile_features.size() > 0) { - feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector(), nz, nx, ny); + feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector(), nz, nx, ny, deduplicate_by_id_set); if (outlayer->features.size() >= feature_limit) { break; } @@ -2221,7 +2245,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int } if (pending_tile_features.size() > 0) { - feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector(), nz, nx, ny); + feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector(), nz, nx, ny, deduplicate_by_id_set); pending_tile_features.clear(); if (outlayer->features.size() >= feature_limit) { break; @@ -2261,7 +2285,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int detail_or_unspecified, buffer, keep, exclude, exclude_prefix, false, NULL, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, - 1, clipbboxes); + 1, clipbboxes, deduplicate_by_id); if (child.size() > 0) { next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y); } diff --git a/geometry.hpp b/geometry.hpp index db96a2cd..73a621ba 100644 --- a/geometry.hpp +++ b/geometry.hpp @@ -147,7 +147,8 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int double tiny_polygon_size, std::vector const &bins, std::string const &bin_by_id_list, std::string const &accumulate_numeric, size_t feature_limit, - std::vector const &clipbboxes); + std::vector const &clipbboxes, + bool deduplicate_by_id); std::string overzoom(std::vector const &tiles, int nz, int nx, int ny, int detail, int buffer, @@ -162,7 +163,8 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int n double tiny_polygon_size, std::vector const &bins, std::string const &bin_by_id_list, std::string const &accumulate_numeric, size_t feature_limit, - std::vector const &clipbboxes); + std::vector const &clipbboxes, + bool deduplicate_by_id); draw center_of_mass_mp(const drawvec &dv); diff --git a/overzoom.cpp b/overzoom.cpp index c7eb922d..7fb4d6fd 100644 --- a/overzoom.cpp +++ b/overzoom.cpp @@ -19,6 +19,7 @@ int detail = 12; // tippecanoe-style: mvt extent == 1 << detail int buffer = 5; // tippecanoe-style: mvt buffer == extent * buffer / 256; bool demultiply = false; bool do_compress = true; +bool deduplicate_by_id = false; std::string filter; bool preserve_input_order = false; @@ -94,6 +95,7 @@ int main(int argc, char **argv) { {"clip-bounding-box", required_argument, 0, 'k' & 0x1F}, {"clip-polygon", required_argument, 0, 'l' & 0x1F}, {"clip-polygon-file", required_argument, 0, 'm' & 0x1F}, + {"deduplicate-by-id", no_argument, 0, 'i' & 0x1F}, {0, 0, 0, 0}, }; @@ -213,6 +215,11 @@ int main(int argc, char **argv) { break; } + case 'i' & 0x1F: { + deduplicate_by_id = true; + break; + } + default: fprintf(stderr, "Unrecognized flag -%c\n", i); usage(argv); @@ -354,7 +361,7 @@ int main(int argc, char **argv) { its.push_back(std::move(t)); } - out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes); + out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes, deduplicate_by_id); } FILE *f = fopen(outfile, "wb"); diff --git a/tests/pbf/1.json b/tests/pbf/1.json new file mode 100644 index 00000000..14e2e105 --- /dev/null +++ b/tests/pbf/1.json @@ -0,0 +1,3 @@ +{"type":"Feature","properties":{"what":"no id"},"geometry":{"type":"Point","coordinates":[0,0]}} +{"type":"Feature","id":12345,"properties":{"what":"will survive"},"geometry":{"type":"Point","coordinates":[0,0]}} +{"type":"Feature","id":12346,"properties":{"what":"will win over the duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}} diff --git a/tests/pbf/2.json b/tests/pbf/2.json new file mode 100644 index 00000000..9506f22f --- /dev/null +++ b/tests/pbf/2.json @@ -0,0 +1,3 @@ +{"type":"Feature","properties":{"what":"no id again"},"geometry":{"type":"Point","coordinates":[0,0]}} +{"type":"Feature","id":12346,"properties":{"what":"will be lost as a duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}} +{"type":"Feature","id":12347,"properties":{"what":"will be added"},"geometry":{"type":"Point","coordinates":[0,0]}} diff --git a/tests/pbf/merged-dedup.pbf.json b/tests/pbf/merged-dedup.pbf.json new file mode 100644 index 00000000..5727adba --- /dev/null +++ b/tests/pbf/merged-dedup.pbf.json @@ -0,0 +1,13 @@ +{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +] } +] } diff --git a/tests/pbf/merged-nodedup.pbf.json b/tests/pbf/merged-nodedup.pbf.json new file mode 100644 index 00000000..dfd65ab2 --- /dev/null +++ b/tests/pbf/merged-nodedup.pbf.json @@ -0,0 +1,15 @@ +{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "id": 12346, "properties": { "what": "will be lost as a duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +, +{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } } +] } +] } diff --git a/tile-join.cpp b/tile-join.cpp index d9484a28..2411fe06 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -982,7 +982,7 @@ struct tileset_reader { false, &next_overzoomed_tiles, false, NULL, false, std::unordered_map(), unidecode_data, 0, 0, std::vector(), "", "", SIZE_MAX, - std::vector()); + std::vector(), false); return ret; } diff --git a/version.hpp b/version.hpp index a240598b..ea243180 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v2.76.0" +#define VERSION "v2.77.0" #endif