Be more careful to retry when the feature count is exceeded (#257)

* Clip before dealing with multiplier or filters in overzoom

* Be more careful to retry when the feature count is exceeded

* Adjust the estimated total feature count for the multiplier too

* Fix the feature count estimates, I think

* Pass build info into the version string

* Report the actual max zoom of any tiles as the metadata maxzoom

* Revert unneeded renaming to make the diff more readable

* Clean up the adjustments to tile sizes and feature counts

* Update version and changelog

* Dropping a feature into a multiplier cluster still effectively drops it

* Update changelog

* Rethink the changelog description

* Don't try to truncate zooms if we are still tiling at z18

CHANGELOG.md

@@ -1,3 +1,11 @@
+# 2.60.0
+
+* Fix bad interaction between --retain-points-multiplier and stopping early when the tile feature limit is reached
+* Fix another bad interaction between --retain-points-multiplier, dropping-as-needed, and variable depth tile pyramids
+* Add optional BUILD_INFO string to version
+* Reorder overzoom logic to clip before dealing with multiplier and filters
+* When --generate-variable-depth-tile-pyramid is in use, report the actual highest zoom generated as tileset maxzoom
+
 # 2.59.0
 
 * Correct `antimeridian_adjusted_bounds` latitude calculation when vertices extend beyond the edge of the Mercator plane

Makefile

@@ -1,14 +1,15 @@
 PREFIX ?= /usr/local
 MANDIR ?= $(PREFIX)/share/man/man1/
 BUILDTYPE ?= Release
+BUILD_INFO ?=
 SHELL = /bin/sh
 
 
 # inherit from env if set
 CC := $(CC)
 CXX := $(CXX)
-CFLAGS := $(CFLAGS) -fPIE
-CXXFLAGS := $(CXXFLAGS) -std=c++17 -fPIE
+CFLAGS := $(CFLAGS) -fPIE -DBUILD_INFO=$(BUILD_INFO)
+CXXFLAGS := $(CXXFLAGS) -std=c++17 -fPIE -DBUILD_INFO=$(BUILD_INFO)
 LDFLAGS := $(LDFLAGS)
 WARNING_FLAGS := -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow
 RELEASE_FLAGS := -O3 -DNDEBUG

clip.cpp

@@ -1188,37 +1188,6 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 			static const std::string retain_points_multiplier_sequence = "tippecanoe:retain_points_multiplier_sequence";
 
 			for (auto feature : layer.features) {
-				bool flush_multiplier_cluster = false;
-				if (demultiply) {
-					for (ssize_t i = feature.tags.size() - 2; i >= 0; i -= 2) {
-						if (layer.keys[feature.tags[i]] == retain_points_multiplier_first) {
-							mvt_value v = layer.values[feature.tags[i + 1]];
-							if (v.type == mvt_bool && v.numeric_value.bool_value) {
-								flush_multiplier_cluster = true;
-								feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
-							}
-						} else if (i < (ssize_t) feature.tags.size() && layer.keys[feature.tags[i]] == retain_points_multiplier_sequence) {
-							mvt_value v = layer.values[feature.tags[i + 1]];
-							feature.seq = mvt_value_to_long_long(v);
-							feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
-						}
-					}
-				} else {
-					flush_multiplier_cluster = true;
-				}
-
-				if (flush_multiplier_cluster) {
-					if (pending_tile_features.size() > 0) {
-						feature_out(pending_tile_features, *outlayer, keep, attribute_accum, tile_stringpool);
-						pending_tile_features.clear();
-					}
-				}
-
-				std::set<std::string> exclude_attributes;
-				if (filter != NULL && !evaluate(feature, layer, filter, exclude_attributes, nz, unidecode_data)) {
-					continue;
-				}
-
 				drawvec geom;
 				int t = feature.type;
 
@@ -1268,6 +1237,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 
 					long long b = outtilesize * buffer / 256;
 					if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) {
+						// quick exclusion by bounding box
 						continue;
 					}
 
@@ -1281,6 +1251,42 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 					}
 				}
 
+				if (geom.size() == 0) {
+					// clipped away
+					continue;
+				}
+
+				bool flush_multiplier_cluster = false;
+				if (demultiply) {
+					for (ssize_t i = feature.tags.size() - 2; i >= 0; i -= 2) {
+						if (layer.keys[feature.tags[i]] == retain_points_multiplier_first) {
+							mvt_value v = layer.values[feature.tags[i + 1]];
+							if (v.type == mvt_bool && v.numeric_value.bool_value) {
+								flush_multiplier_cluster = true;
+								feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
+							}
+						} else if (i < (ssize_t) feature.tags.size() && layer.keys[feature.tags[i]] == retain_points_multiplier_sequence) {
+							mvt_value v = layer.values[feature.tags[i + 1]];
+							feature.seq = mvt_value_to_long_long(v);
+							feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
+						}
+					}
+				} else {
+					flush_multiplier_cluster = true;
+				}
+
+				if (flush_multiplier_cluster) {
+					if (pending_tile_features.size() > 0) {
+						feature_out(pending_tile_features, *outlayer, keep, attribute_accum, tile_stringpool);
+						pending_tile_features.clear();
+					}
+				}
+
+				std::set<std::string> exclude_attributes;
+				if (filter != NULL && !evaluate(feature, layer, filter, exclude_attributes, nz, unidecode_data)) {
+					continue;
+				}
+
 				bool still_need_simplification_after_reduction = false;
 				if (t == VT_POLYGON && tiny_polygon_size > 0) {
 					bool simplified_away_by_reduction = false;

main.cpp

@@ -49,7 +49,6 @@
 #include "tile.hpp"
 #include "pool.hpp"
 #include "projection.hpp"
-#include "version.hpp"
 #include "memfile.hpp"
 #include "main.hpp"
 #include "geojson.hpp"
@@ -3608,7 +3607,7 @@ int main(int argc, char **argv) {
 		}
 
 		case 'v':
-			fprintf(stderr, "tippecanoe %s\n", VERSION);
+			fprintf(stderr, "tippecanoe %s\n", version_str().c_str());
 			exit(EXIT_SUCCESS);
 
 		case 'P':

mbtiles.cpp

@@ -654,6 +654,17 @@ static double sixdig(double val) {
 	return std::round(val * 1e6) / 1e6;
 }
 
+#define str(x) #x
+#define xstr(x) str(x)
+std::string version_str() {
+	std::string s = VERSION;
+	std::string build_info = xstr(BUILD_INFO);
+	if (build_info.size() > 0) {
+		s += " " + build_info;
+	}
+	return s;
+}
+
 metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies, int basezoom, double droprate, int retain_points_multiplier) {
 	metadata m;
 
@@ -684,7 +695,7 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
 		m.attribution = attribution;
 	}
 
-	m.generator = program + " " + VERSION;
+	m.generator = program + " " + version_str();
 	m.generator_options = commandline;
 
 	m.strategies_json = stringify_strategies(strategies);

mbtiles.hpp

@@ -77,5 +77,6 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
 std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps, bool trunc);
 
 void add_to_tilestats(std::map<std::string, tilestat> &tilestats, std::string const &layername, serial_val const &val);
+std::string version_str();
 
 #endif

tests/loop/out/-z0_-O200_--cluster-densest-as-needed.json

@@ -9,7 +9,7 @@
 "maxzoom": "0",
 "minzoom": "0",
 "name": "tests/loop/out/-z0_-O200_--cluster-densest-as-needed.json.check.mbtiles",
-"strategies": "[{\"coalesced_as_needed\":999,\"feature_count_desired\":201}]",
+"strategies": "[{\"coalesced_as_needed\":999,\"feature_count_desired\":1000}]",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/loop/out/-z0_-O200_--drop-densest-as-needed.json

@@ -9,7 +9,7 @@
 "maxzoom": "0",
 "minzoom": "0",
 "name": "tests/loop/out/-z0_-O200_--drop-densest-as-needed.json.check.mbtiles",
-"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":201}]",
+"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":1000}]",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/loop/out/-z0_-O200_--drop-fraction-as-needed.json

@@ -9,7 +9,7 @@
 "maxzoom": "0",
 "minzoom": "0",
 "name": "tests/loop/out/-z0_-O200_--drop-fraction-as-needed.json.check.mbtiles",
-"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":201}]",
+"strategies": "[{\"dropped_as_needed\":999,\"feature_count_desired\":1000}]",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/raw-tiles/nothing.json

@@ -1,14 +1,15 @@
 { "type": "FeatureCollection", "properties": {
 "antimeridian_adjusted_bounds": "0.000000,85.051129,0.000000,85.051129",
 "bounds": "-180.000000,85.051129,180.000000,85.051129",
-"center": "-179.989014,85.051129,14",
+"center": "0.000000,85.051129,0",
 "description": "tests/raw-tiles/nothing",
 "format": "pbf",
 "generator_options": "./tippecanoe -q -f -e tests/raw-tiles/nothing tests/raw-tiles/nothing.geojson",
-"json": "{\"vector_layers\":[{\"id\":\"nothing\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":14,\"fields\":{}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"nothing\",\"count\":1,\"geometry\":\"Point\",\"attributeCount\":0,\"attributes\":[]}]}}",
-"maxzoom": "14",
+"json": "{\"vector_layers\":[{\"id\":\"nothing\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":0,\"fields\":{}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"nothing\",\"count\":1,\"geometry\":\"Point\",\"attributeCount\":0,\"attributes\":[]}]}}",
+"maxzoom": "0",
 "minzoom": "0",
 "name": "tests/raw-tiles/nothing",
+"tippecanoe_decisions": "{\"basezoom\":14,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tile.cpp

@@ -1525,6 +1525,7 @@ bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features
 	ssize_t which_serial_feature;
 
 	if (find_feature_to_accumulate_onto(layer.features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) {
+		strategy.dropped_as_needed++;
 		if (layer.multiplier_cluster_size < (size_t) retain_points_multiplier) {
 			// we have capacity to keep this feature as part of an existing multiplier cluster that isn't full yet
 			// so do that instead of dropping it
@@ -1532,7 +1533,6 @@ bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features
 			return false;  // converted rather than dropped
 		} else {
 			preserve_attributes(attribute_accum, sf, layer.features[which_serial_feature]);
-			strategy.dropped_as_needed++;
 			drop_rest = true;
 			return true;  // dropped
 		}
@@ -1589,8 +1589,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 		// empirical estimate from ne_10m_admin_0_countries, CPAD units, Cal fires.
 		// only try to make an overzoomable final tile if it seems like it might work
 		long long estimated_output_tile_size = 0.6693 * estimated_complexity - 3.36e+04;
-
-		if (estimated_output_tile_size < (long long) (0.9 * max_tile_size)) {
+		if (estimated_output_tile_size < (long long) (0.9 * max_tile_size) && 30 - z > detail) {
 			first_detail = 30 - z;
 			second_detail = detail;
 			trying_to_stop_early = true;
@@ -1640,6 +1639,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 		size_t lead_features_count = 0;			     // of the tile so far
 		size_t other_multiplier_cluster_features_count = 0;  // of the tile so far
 
+		bool too_many_features = false;
+		bool too_many_bytes = false;
+
 		std::atomic<bool> within[child_shards];
 		long long start_geompos[child_shards];
 		for (size_t i = 0; i < (size_t) child_shards; i++) {
@@ -1867,8 +1869,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				} else if (additional[A_DROP_DENSEST_AS_NEEDED]) {
 					add_sample_to(gaps, sf.gap, gaps_increment, seq);
 					if (sf.gap < mingap) {
-						if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
 						can_stop_early = false;
+						if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
 							continue;
 						}
 					}
@@ -1912,8 +1914,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					// search here is for LLONG_MAX, not minextent, because we are dropping features, not coalescing them,
 					// so we shouldn't expect to find anything small that we can related this feature to.
 					if (minextent != 0 && sf.extent + coalesced_area <= minextent) {
-						if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
 						can_stop_early = false;
+						if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
 							continue;
 						}
 					}
@@ -1931,11 +1933,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					}
 				} else if (additional[A_DROP_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP]) {
 					add_sample_to(drop_sequences, drop_sequence, drop_sequences_increment, seq);
-					// search here is for LLONG_MAX, not minextent, because we are dropping features, not coalescing them,
-					// so we shouldn't expect to find anything small that we can related this feature to.
 					if (mindrop_sequence != 0 && drop_sequence <= mindrop_sequence) {
-						if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
 						can_stop_early = false;
+						if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
 							continue;
 						}
 					}
@@ -1995,10 +1995,42 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			unsigned long long sfindex = sf.index;
 
 			if (sf.geometry.size() > 0) {
-				if (lead_features_count > max_tile_size || (lead_features_count + other_multiplier_cluster_features_count > max_tile_features && !prevent[P_FEATURE_LIMIT])) {
+				// There are two adjustments that we need to make
+				// when determining if we have hit the feature count
+				// or byte size limit:
+				//
+				// The max_tile_size and max_tile_features are inflated
+				// to account for the number of multiplier clusters features
+				// we are carrying around in addition to their lead features.
+
+				size_t adjusted_max_tile_size = max_tile_size;
+				if (lead_features_count > 0) {
+					adjusted_max_tile_size = adjusted_max_tile_size * (lead_features_count + other_multiplier_cluster_features_count) / lead_features_count;
+				}
+				size_t adjusted_max_tile_features = max_tile_features;
+				if (lead_features_count > 0) {
+					adjusted_max_tile_features = adjusted_max_tile_features * (lead_features_count + other_multiplier_cluster_features_count) / lead_features_count;
+				}
+
+				// The number of features in the tile, meanwhile, is inflated
+				// to account for the number of features that we have skipped over
+				// because we were already over the limit, in addition to those
+				// that we have actually added to the layer (as either lead or
+				// multiplier features).
+
+				size_t adjusted_feature_count = lead_features_count + other_multiplier_cluster_features_count;
+				if (kept > 0) {
+					adjusted_feature_count = adjusted_feature_count * (skipped + kept) / kept;
+				}
+
+				if (too_many_bytes || adjusted_feature_count > adjusted_max_tile_size) {
 					// Even being maximally conservative, each feature is still going to be
 					// at least one byte in the output tile, so this can't possibly work.
 					skipped++;
+					too_many_bytes = true;
+				} else if (too_many_features || ((adjusted_feature_count > adjusted_max_tile_features) && !prevent[P_FEATURE_LIMIT])) {
+					skipped++;
+					too_many_features = true;
 				} else {
 					kept++;
 
@@ -2134,13 +2166,13 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 		first_time = false;
 
 		// Adjust tile size limit based on the ratio of multiplier cluster features to lead features
-		size_t scaled_max_tile_size = max_tile_size;
+		size_t adjusted_max_tile_size = max_tile_size;
 		if (lead_features_count > 0) {
-			scaled_max_tile_size *= (lead_features_count + other_multiplier_cluster_features_count) / lead_features_count;
+			adjusted_max_tile_size = adjusted_max_tile_size * (lead_features_count + other_multiplier_cluster_features_count) / lead_features_count;
 		}
-		size_t scaled_max_tile_features = max_tile_features;
+		size_t adjusted_max_tile_features = max_tile_features;
 		if (lead_features_count > 0) {
-			scaled_max_tile_features *= (lead_features_count + other_multiplier_cluster_features_count) / lead_features_count;
+			adjusted_max_tile_features = adjusted_max_tile_features * (lead_features_count + other_multiplier_cluster_features_count) / lead_features_count;
 		}
 
 		// Operations on the features within each layer:
@@ -2387,11 +2419,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 		}
 
 		mvt_tile tile;
-		size_t totalsize = 0;
+		size_t feature_count = 0;
 
 		for (auto layer_iterator = layers.begin(); layer_iterator != layers.end(); ++layer_iterator) {
 			std::vector<serial_feature> &layer_features = layer_iterator->second.features;
-			totalsize += layer_features.size();
+			feature_count += layer_features.size();
 
 			mvt_layer layer;
 			layer.name = layer_iterator->first;
@@ -2482,14 +2514,23 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			continue;
 		}
 
-		if (totalsize > 0 && tile.layers.size() > 0) {
-			if (totalsize > scaled_max_tile_features && !prevent[P_FEATURE_LIMIT]) {
-				if (totalsize > arg->feature_count_out) {
-					arg->feature_count_out = totalsize;
+		// Again, adjust the retabulated feature count to estimate
+		// how many total features there would have been if we hadn't
+		// hit the limit and started dropping early.
+
+		size_t adjusted_feature_count = feature_count;
+		if (kept > 0) {
+			adjusted_feature_count = adjusted_feature_count * (skipped + kept) / kept;
+		}
+
+		if (adjusted_feature_count > 0 && tile.layers.size() > 0) {
+			if (too_many_features || (adjusted_feature_count > adjusted_max_tile_features && !prevent[P_FEATURE_LIMIT])) {
+				if (adjusted_feature_count > arg->feature_count_out) {
+					arg->feature_count_out = adjusted_feature_count;
 				}
 
 				if (!quiet) {
-					fprintf(stderr, "tile %d/%u/%u has %zu features, >%zu    \n", z, tx, ty, totalsize, scaled_max_tile_features);
+					fprintf(stderr, "tile %d/%u/%u has %zu (estimated %zu) features, >%zu    \n", z, tx, ty, feature_count, adjusted_feature_count, adjusted_max_tile_features);
 				}
 
 				if (trying_to_stop_early && line_detail == first_detail) {
@@ -2515,7 +2556,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					line_detail++;	// to keep it the same when the loop decrements it
 					continue;
 				} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
-					mingap_fraction = mingap_fraction * scaled_max_tile_features / totalsize * 0.80;
+					mingap_fraction = mingap_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.80;
 					unsigned long long m = choose_mingap(gaps, mingap_fraction, mingap);
 					if (m != mingap) {
 						mingap = m;
@@ -2530,7 +2571,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						continue;
 					}
 				} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
-					minextent_fraction = minextent_fraction * scaled_max_tile_features / totalsize * 0.75;
+					minextent_fraction = minextent_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.75;
 					long long m = choose_minextent(extents, minextent_fraction, minextent);
 					if (m != minextent) {
 						minextent = m;
@@ -2544,11 +2585,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						line_detail++;
 						continue;
 					}
-				} else if (totalsize > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
+				} else if (feature_count > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
 					// The 95% is a guess to avoid too many retries
 					// and probably actually varies based on how much duplicated metadata there is
 
-					mindrop_sequence_fraction = mindrop_sequence_fraction * scaled_max_tile_features / totalsize * 0.95;
+					mindrop_sequence_fraction = mindrop_sequence_fraction * adjusted_max_tile_features / adjusted_feature_count * 0.95;
 					unsigned long long m = choose_mindrop_sequence(drop_sequences, mindrop_sequence_fraction, mindrop_sequence);
 					if (m != mindrop_sequence) {
 						mindrop_sequence = m;
@@ -2581,26 +2622,24 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				compressed = pbf;
 			}
 
-			if (trying_to_stop_early && line_detail == first_detail) {
-				// printf("%lld %zu\n", estimated_complexity, compressed.size());
+			// And similarly, adjust the compressed byte size to estimate
+			// what it would have been if we hadn't stopped dropping features early
+
+			size_t adjusted_tile_size = compressed.size();
+			if (kept > 0) {
+				adjusted_tile_size = adjusted_tile_size * (kept + skipped) / kept;
 			}
 
-			if (compressed.size() > scaled_max_tile_size && !prevent[P_KILOBYTE_LIMIT]) {
-				// Estimate how big it really should have been compressed
-				// from how many features were kept vs skipped for already being
-				// over the threshold
-
-				double kept_adjust = (skipped + kept) / (double) kept;
-
-				if (compressed.size() > arg->tile_size_out) {
-					arg->tile_size_out = compressed.size() * kept_adjust;
+			if (too_many_bytes || (adjusted_tile_size > adjusted_max_tile_size && !prevent[P_KILOBYTE_LIMIT])) {
+				if (adjusted_tile_size > arg->tile_size_out) {
+					arg->tile_size_out = adjusted_tile_size;
 				}
 
 				if (!quiet) {
-					if (skipped > 0) {
-						fprintf(stderr, "tile %d/%u/%u size is %lld (probably really %lld) with detail %d, >%zu    \n", z, tx, ty, (long long) compressed.size(), (long long) (compressed.size() * kept_adjust), line_detail, scaled_max_tile_size);
+					if (adjusted_tile_size == compressed.size()) {
+						fprintf(stderr, "tile %d/%u/%u size is %lld (probably really %zu) with detail %d, >%zu    \n", z, tx, ty, (long long) compressed.size(), adjusted_tile_size, line_detail, adjusted_max_tile_size);
 					} else {
-						fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >%zu    \n", z, tx, ty, (long long) compressed.size(), line_detail, scaled_max_tile_size);
+						fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >%zu    \n", z, tx, ty, (long long) compressed.size(), line_detail, adjusted_max_tile_size);
 					}
 				}
 
@@ -2627,7 +2666,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					}
 					line_detail++;	// to keep it the same when the loop decrements it
 				} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
-					mingap_fraction = mingap_fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.80;
+					mingap_fraction = mingap_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.80;
 					unsigned long long m = choose_mingap(gaps, mingap_fraction, mingap);
 					if (m != mingap) {
 						mingap = m;
@@ -2642,7 +2681,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						continue;
 					}
 				} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
-					minextent_fraction = minextent_fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.75;
+					minextent_fraction = minextent_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75;
 					long long m = choose_minextent(extents, minextent_fraction, minextent);
 					if (m != minextent) {
 						minextent = m;
@@ -2656,8 +2695,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						line_detail++;
 						continue;
 					}
-				} else if (totalsize > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
-					mindrop_sequence_fraction = mindrop_sequence_fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.75;
+				} else if (feature_count > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
+					mindrop_sequence_fraction = mindrop_sequence_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75;
 					unsigned long long m = choose_mindrop_sequence(drop_sequences, mindrop_sequence_fraction, mindrop_sequence);
 					if (m != mindrop_sequence) {
 						mindrop_sequence = m;
@@ -2682,6 +2721,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					exit(EXIT_PTHREAD);
 				}
 
+				if (skipped > 0 || too_many_bytes || too_many_features) {
+					fprintf(stderr, "Can't happen: writing tile even though we skipped\n");
+					exit(EXIT_IMPOSSIBLE);
+				}
+
 				if (outdb != NULL) {
 					mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
 				} else if (outdir != NULL) {
@@ -2802,7 +2846,6 @@ exit(EXIT_IMPOSSIBLE);
 				dc.begin();
 			}
 
-			arg->wrote_zoom = z;
 			long long len;
 
 			struct zxy parent(z - 1, x / 2, y / 2);
@@ -2810,6 +2853,7 @@ exit(EXIT_IMPOSSIBLE);
 				skip_tile(&dc, &geompos, arg->compressed);
 				len = 1;
 			} else {
+				arg->wrote_zoom = z;
 				len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out);
 			}
 
@@ -2903,6 +2947,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 	std::set<zxy> skip_children;
 
 	int z;
+	int largest_written = -1;
+
 	for (z = iz; z <= maxzoom; z++) {
 		std::atomic<long long> most(0);
 
@@ -3122,6 +3168,9 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 				if (args[thread].wrote_zoom > z) {
 					z = args[thread].wrote_zoom;
 				}
+				if (args[thread].wrote_zoom > largest_written) {
+					largest_written = args[thread].wrote_zoom;
+				}
 
 				if (args[thread].still_dropping) {
 					extend_zooms = true;
@@ -3184,6 +3233,10 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 		}
 	}
 
+	if (largest_written >= 0 && maxzoom > largest_written) {
+		maxzoom = largest_written;
+	}
+
 	for (size_t j = 0; j < TEMP_FILES; j++) {
 		// Can be < 0 if there is only one source file, at z0
 		if (geomfd[j] >= 0) {

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.59.0"
+#define VERSION "v2.60.0"
 
 #endif