Merge remote-tracking branch 'origin/main' into faster-bin-assign

CHANGELOG.md

@@ -1,3 +1,18 @@
+# 2.78.0
+
+* Fix potential infinite loops in as-needed dropping and coalescing.
+  When the threshold cannot be increased, it is now an error, rather than
+  falling back to trying to lower the detail.
+* Cleaning of complex polygon geometries now happens in stages
+  to avoid performance problems when there are very large numbers
+  of vertices.
+* Label point generation happens earlier in tiling, to avoid doing slow
+  operations on polygons that will not be retained anyway.
+
+# 2.77.0
+
+* Add --deduplicate-by-id option to tippecanoe-overzoom
+
 # 2.76.0
 
 * Add missing case for accumulating the mean of attributes that are inconsistently present

MADE_WITH.md

@@ -1,19 +1,19 @@
-## [Visualizing a Month of Lightning](http://rousseau.io/2015/03/23/visualizing-a-month-of-lightning) by Jordan Rousseau
+## [Visualizing a Month of Lightning](https://medium.com/@jvrousseau/visualizing-a-month-of-lightning-8ab63d040d8c) by Jordan Rousseau
 
 ![](http://rousseau.io/assets/img/ltg-studio-style.png)
 
-## [Making the most detailed tweet map ever](https://www.mapbox.com/blog/twitter-map-every-tweet/) by Eric Fischer
+## [Making the most detailed tweet map ever](https://web.archive.org/web/20161220144946/https://www.mapbox.com/blog/twitter-map-every-tweet/) by Eric Fischer
 
 ![](https://farm8.staticflickr.com/7505/15869589271_8a02e84c24_b.jpg)
 
-## [Superpowering Runkeeper's 1.5 million walks, runs, and bike rides](https://www.mapbox.com/blog/runkeeper-million-routes/)
+## [Superpowering Runkeeper's 1.5 million walks, runs, and bike rides](https://web.archive.org/web/20161219225305/https://www.mapbox.com/blog/runkeeper-million-routes/)
 
 ![](https://c1.staticflickr.com/9/8605/15852245980_1ecf0894b8_b.jpg)
 
-## [The Geotaggers' World Atlas](https://www.mapbox.com/blog/geotaggers-world-atlas/) by Eric Fischer
+## [The Geotaggers' World Atlas](https://web.archive.org/web/20170319084321/https://www.mapbox.com/blog/geotaggers-world-atlas/) by Eric Fischer
 
 ![](http://farm8.staticflickr.com/7634/17040546408_0a14752e6d_b.jpg)
 
-## [Atmospheric River](https://www.mapbox.com/blog/atmospheric-river/)
+## [Atmospheric River](https://web.archive.org/web/20161011171030/https://www.mapbox.com/blog/atmospheric-river/)
 
 ![](http://farm9.staticflickr.com/8630/16253097589_4dfc706b22_b.jpg)

Makefile

@@ -443,6 +443,16 @@ overzoom-test: tippecanoe-overzoom
 	./tippecanoe-overzoom -o tests/pbf/squirrels-13-2413-3077-clip.pbf --clip-polygon-file tests/pbf/squirrels-clip.json tests/pbf/squirrels-13-2413-3077.pbf 13/2413/3077 13/2413/3077
 	cmp tests/pbf/squirrels-13-2413-3077-clip.pbf /dev/null  # clipped away
 	rm tests/pbf/squirrels-13-2413-3077-clip.pbf
+	# Deduplication by feature ID
+	./tippecanoe -z0 -f -e tests/pbf/1.json.dir -l layer tests/pbf/1.json
+	./tippecanoe -z0 -f -e tests/pbf/2.json.dir -l layer tests/pbf/2.json
+	./tippecanoe-overzoom -o tests/pbf/merged-nodedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0
+	./tippecanoe-decode tests/pbf/merged-nodedup.pbf 0 0 0 > tests/pbf/merged-nodedup.pbf.json.check
+	cmp tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-nodedup.pbf.json
+	./tippecanoe-overzoom --deduplicate-by-id -o tests/pbf/merged-dedup.pbf -t 0/0/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0
+	./tippecanoe-decode tests/pbf/merged-dedup.pbf 0 0 0 > tests/pbf/merged-dedup.pbf.json.check
+	cmp tests/pbf/merged-dedup.pbf.json.check tests/pbf/merged-dedup.pbf.json
+	rm -r tests/pbf/1.json.dir tests/pbf/2.json.dir tests/pbf/merged-nodedup.pbf tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-dedup.pbf.json.check
 
 join-test: tippecanoe tippecanoe-decode tile-join
 	./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}'

README.md

@@ -475,7 +475,7 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
  * `-K` _distance_ or `--cluster-distance=`_distance_: Cluster points (as with `--cluster-densest-as-needed`, but without the experimental discovery process) that are approximately within _distance_ of each other. The units are tile coordinates within a nominally 256-pixel tile, so the maximum value of 255 allows only one feature per tile. Values around 10 are probably appropriate for typical marker sizes. See `--cluster-densest-as-needed` below for behavior.
  * `-k` _zoom_ or `--cluster-maxzoom=`_zoom_: Max zoom on which to cluster points if clustering is enabled.
  * `-kg` or `--cluster-maxzoom=g`: Set `--cluster-maxzoom=` to `maxzoom - 1` so that all features are visible at the maximum zoom level.
- * `--preserve-point-density-threshold=`_level_: At the low zoom levels, do not reduce point density below the specified _level_, even if the specfied drop rate would normally call for it, so that low-density areas of the map do not appear blank. The unit is the distance between preserved points, as a fraction of the size of a tile. Values of 32 or 64 are probably appropriate for typical marker sizes.
+ * `--preserve-point-density-threshold=`_level_: At the low zoom levels, do not reduce point density below the specified _level_, even if the specified drop rate would normally call for it, so that low-density areas of the map do not appear blank. The unit is the distance between preserved points, as a fraction of the size of a tile. Values of 32 or 64 are probably appropriate for typical marker sizes.
 
 ### Dropping a fraction of features to keep under tile size limits
 
@@ -719,7 +719,7 @@ and perhaps
     make install
 
 Tippecanoe now requires features from the 2011 C++ standard. If your compiler is older than
-that, you will need to install a newer one. On MacOS, updating to the lastest XCode should
+that, you will need to install a newer one. On MacOS, updating to the latest XCode should
 get you a new enough version of `clang++`. On Linux, you should be able to upgrade `g++` with
 
 ```

clip.cpp

@@ -571,8 +571,7 @@ drawvec remove_noop(drawvec geom, int type, int shift) {
 				}
 
 				if (geom[i + 1].op == VT_CLOSEPATH) {
-					// followed by closepath: not possible
+					// followed by closepath: only possible after close_poly()
-					fprintf(stderr, "Shouldn't happen\n");
 					i++;  // also remove unused closepath
 					continue;
 				}
@@ -1269,7 +1268,8 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
 		     std::string const &accumulate_numeric, size_t feature_limit,
-		     std::vector<clipbbox> const &clipbboxes) {
+		     std::vector<clipbbox> const &clipbboxes,
+		     bool deduplicate_by_id) {
 	std::vector<source_tile> decoded;
 
 	for (auto const &t : tiles) {
@@ -1295,7 +1295,7 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
 		decoded.push_back(out);
 	}
 
-	return overzoom(decoded, nz, nx, ny, detail_or_unspecified, buffer, keep, exclude, exclude_prefix, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, feature_limit, clipbboxes);
+	return overzoom(decoded, nz, nx, ny, detail_or_unspecified, buffer, keep, exclude, exclude_prefix, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, feature_limit, clipbboxes, deduplicate_by_id);
 }
 
 // like a minimal serial_feature, but with mvt_feature-style attributes
@@ -1499,7 +1499,8 @@ static bool feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
 			std::unordered_map<std::string, attribute_op> const &attribute_accum,
 			std::string const &accumulate_numeric,
 			key_pool &key_pool, int buffer, bool include_nonaggregate,
-			std::vector<clipbbox> const &clipbboxes, int nz, int nx, int ny) {
+			std::vector<clipbbox> const &clipbboxes, int nz, int nx, int ny,
+			std::set<unsigned long long> *deduplicate_ids) {
 	// Add geometry to output feature
 
 	drawvec geom = features[0].geom;
@@ -1582,6 +1583,14 @@ static bool feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
 
 	// ID and attributes, if it didn't get clipped away
 
+	if (features[0].has_id && deduplicate_ids != NULL) {
+		if (deduplicate_ids->find(features[0].id) != deduplicate_ids->end()) {
+			outfeature.geometry.clear();
+		} else {
+			deduplicate_ids->insert(features[0].id);
+		}
+	}
+
 	if (outfeature.geometry.size() > 0) {
 		if (features[0].has_id) {
 			outfeature.has_id = true;
@@ -2001,7 +2010,7 @@ mvt_tile assign_to_bins(mvt_tile &features,
 			if (feature_out(outfeatures[i], outlayer,
 					keep, exclude, exclude_prefix, attribute_accum,
 					accumulate_numeric, key_pool, buffer, true,
-					clipbboxes, z, x, y)) {
+					clipbboxes, z, x, y, NULL)) {
 				mvt_feature &nfeature = *outlayer.features.back();
 				mvt_value val;
 				val.type = mvt_uint;
@@ -2038,10 +2047,14 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
 		     std::string const &accumulate_numeric, size_t feature_limit,
-		     std::vector<clipbbox> const &clipbboxes) {
+		     std::vector<clipbbox> const &clipbboxes,
+		     bool deduplicate_by_id) {
 	mvt_tile outtile;
 	key_pool key_pool;
 
+	// map from layer name to ids used in that layer
+	std::map<std::string, std::set<unsigned long long>> deduplicate_ids;
+
 	for (auto const &tile : tiles) {
 		for (auto const &layer : tile.tile.layers) {
 			mvt_layer *outlayer = NULL;
@@ -2068,6 +2081,16 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 				outlayer = &outtile.layers.back();
 			}
 
+			std::set<unsigned long long> *deduplicate_by_id_set = NULL;
+			if (deduplicate_by_id) {
+				auto layer_deduplicate_ids = deduplicate_ids.find(layer.name);
+				if (layer_deduplicate_ids == deduplicate_ids.end()) {
+					deduplicate_ids.emplace(layer.name, std::set<unsigned long long>());
+					layer_deduplicate_ids = deduplicate_ids.find(layer.name);
+				}
+				deduplicate_by_id_set = &layer_deduplicate_ids->second;
+			}
+
 			std::vector<tile_feature> pending_tile_features;
 			double accum_area = 0;
 
@@ -2190,7 +2213,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 
 				if (flush_multiplier_cluster) {
 					if (pending_tile_features.size() > 0) {
-						feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector<clipbbox>(), nz, nx, ny);
+						feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector<clipbbox>(), nz, nx, ny, deduplicate_by_id_set);
 						if (outlayer->features.size() >= feature_limit) {
 							break;
 						}
@@ -2250,7 +2273,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 			}
 
 			if (pending_tile_features.size() > 0) {
-				feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector<clipbbox>(), nz, nx, ny);
+				feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1, bins.size() == 0, std::vector<clipbbox>(), nz, nx, ny, deduplicate_by_id_set);
 				pending_tile_features.clear();
 				if (outlayer->features.size() >= feature_limit) {
 					break;
@@ -2290,7 +2313,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 								     detail_or_unspecified, buffer, keep, exclude, exclude_prefix, false, NULL,
 								     demultiply, filter, preserve_input_order, attribute_accum, unidecode_data,
 								     simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric,
-								     1, clipbboxes);
+								     1, clipbboxes, deduplicate_by_id);
 					if (child.size() > 0) {
 						next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y);
 					}
@@ -2474,3 +2497,88 @@ drawvec fix_polygon(const drawvec &geom, bool use_winding, bool reverse_winding)
 
 	return out;
 }
+
+bool line_is_too_small(drawvec const &geometry, int z, int detail) {
+	if (geometry.size() == 0) {
+		return true;
+	}
+
+	long long x = 0, y = 0;
+	for (auto &g : geometry) {
+		if (g.op == VT_MOVETO) {
+			x = std::llround((double) g.x / (1LL << (32 - detail - z)));
+			y = std::llround((double) g.y / (1LL << (32 - detail - z)));
+		} else {
+			long long xx = std::llround((double) g.x / (1LL << (32 - detail - z)));
+			long long yy = std::llround((double) g.y / (1LL << (32 - detail - z)));
+
+			if (xx != x || yy != y) {
+				return false;
+			}
+		}
+	}
+
+	return true;
+}
+
+void coalesce_polygon(drawvec &geom, bool scale_up) {
+	// wagyu should be able to straightforwardly handle
+	// anything under a few hundred thousand vertices
+	if (geom.size() < 100000) {
+		geom = clean_or_clip_poly(geom, 0, 0, false, scale_up);
+		return;
+	}
+
+	// These geometries were assembled in geometric order,
+	// so sub-batches of them should hopefully union into
+	// reasonable sets.
+	//
+	// Find the first outer ring after halfway point.
+
+	for (size_t i = geom.size() / 2; i < geom.size(); i++) {
+		if (geom[i].op == VT_MOVETO) {
+			size_t j;
+			for (j = i + 1; j < geom.size(); j++) {
+				if (geom[j].op != VT_LINETO) {
+					break;
+				}
+			}
+
+			if (get_area(geom, i, j) > 0) {
+				// If we have an outer ring, split there
+				// and coalesce the two halves
+
+				// Copy second half to new vector
+				std::vector<draw> geom2;
+				geom2.resize(geom.size() - i);
+				for (size_t k = i; k < geom.size(); k++) {
+					geom2[k - i] = geom[k];
+				}
+
+				// Resize vector to include only first half
+				geom.resize(i);
+
+				// Clean each half individually
+				coalesce_polygon(geom, scale_up);
+				coalesce_polygon(geom2, scale_up);
+
+				// Copy second half back with first
+				size_t brk = geom.size();
+				geom.resize(brk + geom2.size());
+				for (size_t k = 0; k < geom2.size(); k++) {
+					geom[brk + k] = geom2[k];
+				}
+				geom2.clear();
+
+				// Clean the combined geometry
+				geom = clean_or_clip_poly(geom, 0, 0, false, scale_up);
+				return;
+			}
+
+			i = j - 1;
+		}
+	}
+
+	// Can't find a breakpoint; take what we can get.
+	geom = clean_or_clip_poly(geom, 0, 0, false, scale_up);
+}

geometry.hpp

@@ -147,7 +147,8 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
 		     std::string const &accumulate_numeric, size_t feature_limit,
-		     std::vector<clipbbox> const &clipbboxes);
+		     std::vector<clipbbox> const &clipbboxes,
+		     bool deduplicate_by_id);
 
 std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int ny,
 		     int detail, int buffer,
@@ -162,7 +163,8 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
 		     std::string const &accumulate_numeric, size_t feature_limit,
-		     std::vector<clipbbox> const &clipbboxes);
+		     std::vector<clipbbox> const &clipbboxes,
+		     bool deduplicate_by_id);
 
 draw center_of_mass_mp(const drawvec &dv);
 
@@ -171,4 +173,7 @@ void get_quadkey_bounds(long long xmin, long long ymin, long long xmax, long lon
 
 clipbbox parse_clip_poly(std::string arg);
 
+bool line_is_too_small(drawvec const &geometry, int z, int detail);
+void coalesce_polygon(drawvec &geom, bool scale_up);
+
 #endif

lambda/README.md

@@ -6,7 +6,7 @@
     sudo yum install git clang sqlite-devel zlib-devel
 
 2. Create a ZIP archive with the `tippecanoe` binary at `bin/tippecanoe` and upload as a Lambda layer
-3. Copy [lambda_function.py](lambda_function.py) (python 3.9 runtime)
+3. Copy [tippecanoe.py](tippecanoe.py) (python 3.9 runtime)
 4. set the environment variable `OUTPUT_BUCKET` to your output bucket; give lambda IAM role access to input/output buckets
 5. Enable ACLs on the output bucket
 

overzoom.cpp

@@ -19,6 +19,7 @@ int detail = 12;  // tippecanoe-style: mvt extent == 1 << detail
 int buffer = 5;	  // tippecanoe-style: mvt buffer == extent * buffer / 256;
 bool demultiply = false;
 bool do_compress = true;
+bool deduplicate_by_id = false;
 
 std::string filter;
 bool preserve_input_order = false;
@@ -94,6 +95,7 @@ int main(int argc, char **argv) {
 		{"clip-bounding-box", required_argument, 0, 'k' & 0x1F},
 		{"clip-polygon", required_argument, 0, 'l' & 0x1F},
 		{"clip-polygon-file", required_argument, 0, 'm' & 0x1F},
+		{"deduplicate-by-id", no_argument, 0, 'i' & 0x1F},
 
 		{0, 0, 0, 0},
 	};
@@ -213,6 +215,11 @@ int main(int argc, char **argv) {
 			break;
 		}
 
+		case 'i' & 0x1F: {
+			deduplicate_by_id = true;
+			break;
+		}
+
 		default:
 			fprintf(stderr, "Unrecognized flag -%c\n", i);
 			usage(argv);
@@ -354,7 +361,7 @@ int main(int argc, char **argv) {
 			its.push_back(std::move(t));
 		}
 
-		out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes);
+		out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes, deduplicate_by_id);
 	}
 
 	FILE *f = fopen(outfile, "wb");

tests/ne_110m_admin_0_countries/out/-z2_--convert-polygons-to-label-points.json

@@ -9,7 +9,6 @@
 "maxzoom": "2",
 "minzoom": "0",
 "name": "tests/ne_110m_admin_0_countries/out/-z2_--convert-polygons-to-label-points.json.check.mbtiles",
-"strategies": "[{},{},{\"tiny_polygons\":1}]",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/pbf/1.json

@@ -0,0 +1,3 @@
+{"type":"Feature","properties":{"what":"no id"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12345,"properties":{"what":"will survive"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12346,"properties":{"what":"will win over the duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}}

tests/pbf/2.json

@@ -0,0 +1,3 @@
+{"type":"Feature","properties":{"what":"no id again"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12346,"properties":{"what":"will be lost as a duplicate"},"geometry":{"type":"Point","coordinates":[0,0]}}
+{"type":"Feature","id":12347,"properties":{"what":"will be added"},"geometry":{"type":"Point","coordinates":[0,0]}}

tests/pbf/merged-dedup.pbf.json

@@ -0,0 +1,13 @@
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+] }

tests/pbf/merged-nodedup.pbf.json

@@ -0,0 +1,15 @@
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "layer", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "what": "no id" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12345, "properties": { "what": "will survive" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12346, "properties": { "what": "will win over the duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "properties": { "what": "no id again" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12346, "properties": { "what": "will be lost as a duplicate" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+,
+{ "type": "Feature", "id": 12347, "properties": { "what": "will be added" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+] }

tile-join.cpp

@@ -982,7 +982,7 @@ struct tileset_reader {
 						   false, &next_overzoomed_tiles, false, NULL, false,
 						   std::unordered_map<std::string, attribute_op>(), unidecode_data, 0, 0,
 						   std::vector<mvt_layer>(), "", "", SIZE_MAX,
-						   std::vector<clipbbox>());
+						   std::vector<clipbbox>(), false);
 			return ret;
 		}
 

tile.cpp

@@ -639,7 +639,7 @@ static double simplify_feature(serial_feature *p, drawvec const &shared_nodes, n
 					// unioned exactly
 					//
 					// don't try to scale up because these are still world coordinates
-					geom = clean_or_clip_poly(geom, 0, 0, false, false);
+					coalesce_polygon(geom, false);
 				}
 
 				// continues to simplify to line_detail even if we have extra detail
@@ -695,7 +695,7 @@ static void *simplification_worker(void *v) {
 
 				if (!a->trying_to_stop_early) {
 					// we can try scaling up because this is now tile scale
-					geom = clean_or_clip_poly(geom, 0, 0, false, true);
+					coalesce_polygon(geom, true);
 					if (additional[A_DEBUG_POLYGON]) {
 						check_polygon(geom);
 					}
@@ -712,12 +712,6 @@ static void *simplification_worker(void *v) {
 			}
 		}
 
-		if (t == VT_POLYGON && additional[A_GENERATE_POLYGON_LABEL_POINTS]) {
-			t = (*features)[i]->t = VT_POINT;
-			geom = checkerboard_anchors(from_tile_scale(geom, z, out_detail), (*features)[i]->tx, (*features)[i]->ty, z, (*features)[i]->label_point);
-			to_tile_scale(geom, z, out_detail);
-		}
-
 		if ((*features)[i]->index == 0) {
 			(*features)[i]->index = i;
 		}
@@ -769,7 +763,7 @@ static unsigned long long choose_mingap(std::vector<unsigned long long> &gaps, d
 	std::stable_sort(gaps.begin(), gaps.end());
 
 	size_t ix = (gaps.size() - 1) * (1 - f);
-	while (ix + 1 < gaps.size() && gaps[ix] == existing_gap) {
+	while (ix + 1 < gaps.size() && gaps[ix] <= existing_gap) {
 		ix++;
 	}
 
@@ -809,7 +803,7 @@ static long long choose_minextent(std::vector<long long> &extents, double f, lon
 	std::stable_sort(extents.begin(), extents.end());
 
 	size_t ix = (extents.size() - 1) * (1 - f);
-	while (ix + 1 < extents.size() && extents[ix] == existing_extent) {
+	while (ix + 1 < extents.size() && extents[ix] <= existing_extent) {
 		ix++;
 	}
 
@@ -824,7 +818,7 @@ static unsigned long long choose_mindrop_sequence(std::vector<unsigned long long
 	std::stable_sort(drop_sequences.begin(), drop_sequences.end());
 
 	size_t ix = (drop_sequences.size() - 1) * (1 - f);
-	while (ix + 1 < drop_sequences.size() && drop_sequences[ix] == existing_drop_sequence) {
+	while (ix + 1 < drop_sequences.size() && drop_sequences[ix] <= existing_drop_sequence) {
 		ix++;
 	}
 
@@ -1551,26 +1545,6 @@ bool find_feature_to_accumulate_onto(std::vector<std::shared_ptr<serial_feature>
 	return false;
 }
 
-static bool line_is_too_small(drawvec const &geometry, int z, int detail) {
-	if (geometry.size() == 0) {
-		return true;
-	}
-
-	long long x = std::round((double) geometry[0].x / (1LL << (32 - detail - z)));
-	long long y = std::round((double) geometry[0].y / (1LL << (32 - detail - z)));
-
-	for (auto &g : geometry) {
-		long long xx = std::round((double) g.x / (1LL << (32 - detail - z)));
-		long long yy = std::round((double) g.y / (1LL << (32 - detail - z)));
-
-		if (xx != x || yy != y) {
-			return false;
-		}
-	}
-
-	return true;
-}
-
 // Keep only a sample of 100K extents for feature dropping,
 // to avoid spending lots of memory on a complete list when there are
 // hundreds of millions of features.
@@ -1885,6 +1859,35 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				extent_previndex = sf.index;
 			}
 
+			// Make label anchors early in tiling, even though it requires simplifying early,
+			// so that if there is no label anchor for this feature in this tile,
+			// we find out now rather than after we have already decided that there
+			// are too_many_bytes.
+			//
+			// label anchors also need to happen before as-needed dropping and coalescing,
+			// so that the geometry type matches for find_feature_to_accumulate_onto.
+			// (or it could happen much later, after all the features are accumulated,
+			// but then it would be too late for too_many_bytes)
+			if (sf.t == VT_POLYGON && additional[A_GENERATE_POLYGON_LABEL_POINTS]) {
+				// exclude features that are invisibly small at this zoom level
+				if (line_is_too_small(sf.geometry, z, line_detail)) {
+					continue;
+				}
+				if (sf.t == VT_POLYGON && get_mp_area(sf.geometry) <= 0) {
+					continue;
+				}
+
+				drawvec ngeom = simplify_lines(sf.geometry, z, tx, ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), sf.simplification, sf.t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, "");
+				if (ngeom.size() == 0) {
+					continue;
+				}
+				sf.geometry = checkerboard_anchors(ngeom, tx, ty, z, sf.label_point);
+				if (sf.geometry.size() == 0) {
+					continue;
+				}
+				sf.t = VT_POINT;
+			}
+
 			unsigned long long drop_sequence = 0;
 			if (additional[A_COALESCE_FRACTION_AS_NEEDED] || additional[A_DROP_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP]) {
 				drop_sequence = calculate_drop_sequence(sf);
@@ -2000,7 +2003,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				} else if (additional[A_COALESCE_DENSEST_AS_NEEDED]) {
 					add_sample_to(gaps, sf.gap, gaps_increment, seq);
 					if (sf.gap < mingap && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) {
-						coalesce_geometry(*features[which_serial_feature], sf);
+						if (sf.t == VT_POINT || !line_is_too_small(sf.geometry, z, line_detail)) {
+							coalesce_geometry(*features[which_serial_feature], sf);
+						}
 						features[which_serial_feature]->coalesced = true;
 						coalesced_area += sf.extent;
 						preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
@@ -2022,7 +2027,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				} else if (additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
 					add_sample_to(extents, sf.extent, extents_increment, seq);
 					if (minextent != 0 && sf.extent + coalesced_area <= minextent && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, minextent)) {
-						coalesce_geometry(*features[which_serial_feature], sf);
+						if (sf.t == VT_POINT || !line_is_too_small(sf.geometry, z, line_detail)) {
+							coalesce_geometry(*features[which_serial_feature], sf);
+						}
 						features[which_serial_feature]->coalesced = true;
 						coalesced_area += sf.extent;
 						preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
@@ -2042,7 +2049,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				} else if (additional[A_COALESCE_FRACTION_AS_NEEDED]) {
 					add_sample_to(drop_sequences, drop_sequence, drop_sequences_increment, seq);
 					if (mindrop_sequence != 0 && drop_sequence <= mindrop_sequence && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) {
-						coalesce_geometry(*features[which_serial_feature], sf);
+						if (sf.t == VT_POINT || !line_is_too_small(sf.geometry, z, line_detail)) {
+							coalesce_geometry(*features[which_serial_feature], sf);
+						}
 						features[which_serial_feature]->coalesced = true;
 						preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
 						strategy.coalesced_as_needed++;
@@ -2201,7 +2210,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 								drawvec to_clean = features[simplified_geometry_through]->geometry;
 
 								// don't scale up because this is still world coordinates
-								to_clean = clean_or_clip_poly(to_clean, 0, 0, false, false);
+								coalesce_polygon(to_clean, false);
 								features[simplified_geometry_through]->geometry = std::move(to_clean);
 							}
 						}
@@ -2470,7 +2479,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					if (layer_features[x]->t == VT_POLYGON) {
 						if (layer_features[x]->coalesced) {
 							// we can try scaling up because this is tile coordinates
-							layer_features[x]->geometry = clean_or_clip_poly(layer_features[x]->geometry, 0, 0, false, true);
+							coalesce_polygon(layer_features[x]->geometry, true);
 						}
 
 						layer_features[x]->geometry = close_poly(layer_features[x]->geometry);
@@ -2501,6 +2510,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					// this is maxzoom; ok to stop early still because they said to limit abruptly
 					layer_features.resize(limit_tile_feature_count_at_maxzoom);
 					too_many_features = false;  // don't try to drop; we have already truncated
+					too_many_bytes = false;	    // don't try to drop; we have already truncated
 					skipped = 0;		    // doesn't matter that we skipped features; we have truncated
 				}
 			} else if (limit_tile_feature_count != 0) {
@@ -2508,6 +2518,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					can_stop_early = false;
 					layer_features.resize(limit_tile_feature_count);
 					too_many_features = false;  // don't try to drop; we have already truncated
+					too_many_bytes = false;	    // don't try to drop; we have already truncated
 					skipped = 0;		    // doesn't matter that we skipped features; we have truncated
 				}
 			}
@@ -2655,8 +2666,14 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					continue;
 				} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
 					mingap_fraction = mingap_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.80;
+					if (mingap_fraction > 0.80) {
+						if (!quiet) {
+							fprintf(stderr, "Need to drop features, but calculated that we should keep %.1f%% of features\n", mingap_fraction * 100.0);
+						}
+						mingap_fraction = 0.80;
+					}
 					unsigned long long m = choose_mingap(gaps, mingap_fraction, mingap);
-					if (m != mingap) {
+					if (m > mingap) {
 						mingap = m;
 						if (mingap > arg->mingap_out) {
 							arg->mingap_out = mingap;
@@ -2667,11 +2684,20 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						}
 						line_detail++;
 						continue;
+					} else {
+						fprintf(stderr, "Can't increase feature gap threshold further\n");
+						exit(EXIT_INCOMPLETE);
 					}
 				} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
 					minextent_fraction = minextent_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.75;
+					if (minextent_fraction > 0.80) {
+						if (!quiet) {
+							fprintf(stderr, "Need to drop features, but calculated that we should keep %.1f%% of features\n", minextent_fraction * 100.0);
+						}
+						minextent_fraction = 0.80;
+					}
 					long long m = choose_minextent(extents, minextent_fraction, minextent);
-					if (m != minextent) {
+					if (m > minextent) {
 						minextent = m;
 						if (minextent > arg->minextent_out) {
 							arg->minextent_out = minextent;
@@ -2682,14 +2708,23 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						}
 						line_detail++;
 						continue;
+					} else {
+						fprintf(stderr, "Can't increase feature area threshold further\n");
+						exit(EXIT_INCOMPLETE);
 					}
 				} else if (feature_count > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
 					// The 95% is a guess to avoid too many retries
 					// and probably actually varies based on how much duplicated metadata there is
 
 					mindrop_sequence_fraction = mindrop_sequence_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.95;
+					if (mindrop_sequence_fraction > 0.80) {
+						if (!quiet) {
+							fprintf(stderr, "Need to drop features, but calculated that we should keep %.1f%% of features\n", mindrop_sequence_fraction * 100.0);
+						}
+						mindrop_sequence_fraction = 0.80;
+					}
 					unsigned long long m = choose_mindrop_sequence(drop_sequences, mindrop_sequence_fraction, mindrop_sequence);
-					if (m != mindrop_sequence) {
+					if (m > mindrop_sequence) {
 						mindrop_sequence = m;
 						if (mindrop_sequence > arg->mindrop_sequence_out) {
 							if (!prevent[P_DYNAMIC_DROP]) {
@@ -2702,6 +2737,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						}
 						line_detail++;	// to keep it the same when the loop decrements it
 						continue;
+					} else {
+						fprintf(stderr, "Can't increase feature count threshold further\n");
+						exit(EXIT_INCOMPLETE);
 					}
 				} else {
 					fprintf(stderr, "Try using --drop-fraction-as-needed or --drop-densest-as-needed.\n");
@@ -2734,7 +2772,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				}
 
 				if (!quiet) {
-					if (adjusted_tile_size == compressed.size()) {
+					if (adjusted_tile_size != compressed.size()) {
 						fprintf(stderr, "tile %d/%u/%u size is %lld (probably really %zu) with detail %d, >%zu    \n", z, tx, ty, (long long) compressed.size(), adjusted_tile_size, line_detail, adjusted_max_tile_size);
 					} else {
 						fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >%zu    \n", z, tx, ty, (long long) compressed.size(), line_detail, adjusted_max_tile_size);
@@ -2765,8 +2803,14 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					line_detail++;	// to keep it the same when the loop decrements it
 				} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
 					mingap_fraction = mingap_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.80;
+					if (mingap_fraction > 0.80) {
+						if (!quiet) {
+							fprintf(stderr, "Need to drop features, but calculated that we should keep %.1f%% of features\n", mingap_fraction * 100.0);
+						}
+						mingap_fraction = 0.80;
+					}
 					unsigned long long m = choose_mingap(gaps, mingap_fraction, mingap);
-					if (m != mingap) {
+					if (m > mingap) {
 						mingap = m;
 						if (mingap > arg->mingap_out) {
 							arg->mingap_out = mingap;
@@ -2777,11 +2821,20 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						}
 						line_detail++;
 						continue;
+					} else {
+						fprintf(stderr, "Can't increase feature gap threshold further\n");
+						exit(EXIT_INCOMPLETE);
 					}
 				} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
 					minextent_fraction = minextent_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75;
+					if (minextent_fraction > 0.80) {
+						if (!quiet) {
+							fprintf(stderr, "Need to drop features, but calculated that we should keep %.1f%% of features\n", minextent_fraction * 100.0);
+						}
+						minextent_fraction = 0.80;
+					}
 					long long m = choose_minextent(extents, minextent_fraction, minextent);
-					if (m != minextent) {
+					if (m > minextent) {
 						minextent = m;
 						if (minextent > arg->minextent_out) {
 							arg->minextent_out = minextent;
@@ -2792,11 +2845,20 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						}
 						line_detail++;
 						continue;
+					} else {
+						fprintf(stderr, "Can't increase feature area threshold further\n");
+						exit(EXIT_INCOMPLETE);
 					}
 				} else if (feature_count > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
 					mindrop_sequence_fraction = mindrop_sequence_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75;
+					if (mindrop_sequence_fraction > 0.80) {
+						if (!quiet) {
+							fprintf(stderr, "Need to drop features, but calculated that we should keep %.1f%% of features\n", mindrop_sequence_fraction * 100.0);
+						}
+						mindrop_sequence_fraction = 0.80;
+					}
 					unsigned long long m = choose_mindrop_sequence(drop_sequences, mindrop_sequence_fraction, mindrop_sequence);
-					if (m != mindrop_sequence) {
+					if (m > mindrop_sequence) {
 						mindrop_sequence = m;
 						if (mindrop_sequence > arg->mindrop_sequence_out) {
 							if (!prevent[P_DYNAMIC_DROP]) {
@@ -2809,6 +2871,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						}
 						line_detail++;
 						continue;
+					} else {
+						fprintf(stderr, "Can't increase feature count threshold further\n");
+						exit(EXIT_INCOMPLETE);
 					}
 				} else {
 					detail_reduced++;

unit.cpp

@@ -152,3 +152,12 @@ TEST_CASE("mvt_geometry bbox") {
 	REQUIRE(start == 0x1c84fc0000000000);
 	REQUIRE(end == 0x1c84ffffffffffff);
 }
+
+TEST_CASE("line_is_too_small") {
+	drawvec dv;
+	dv.emplace_back(VT_MOVETO, 4243099709, 2683872952);
+	dv.emplace_back(VT_LINETO, 4243102487, 2683873977);
+	dv.emplace_back(VT_MOVETO, -51867587, 2683872952);
+	dv.emplace_back(VT_LINETO, -51864809, 2683873977);
+	REQUIRE(line_is_too_small(dv, 0, 10));
+}

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.76.0"
+#define VERSION "v2.78.0"
 
 #endif