Add point retention multiplier (#179)

* Add an option to retain N times as many points as usual at each zoom

* Tests for point multipler with specified and guessed maxzooms

* Work in progress on inverse spatial ordering

* Fix inverse spatial feature order

* --reorder was depending on a feature index that wasn't being preserved

* Separate ordering by feature_minzoom from ordering inverse-spatially

* Add a test for the inverse spatial ordering

* Store the basezoom/droprate/multiplier decisions in tileset metadata

* Progress on adding filters to tippecanoe-overzoom

* Type promotion for comparison

* Look up the attribute value for ordering

* Add test of thinning and ordering features

* Plumb tippecanoe_decisions metadata through pmtiles

* Be careful not to put infinities in JSON

* Fix accidental dropping in what is meant to preserve sparse points

* Start distinguishing true, false, and null in expressions

* Most of the type conversions

* Add boolean conversions

* Literals and conjunctions

* Add filtering to tippecanoe-overzoom

* Add a test of filtering in overzoom

* Fix boolean conjunctions

* Handle the combination of cluster size and filtering

* Rework dot dropping to reconcile density threshold and multiplier

* Revert "Rework dot dropping to reconcile density threshold and multiplier"

This reverts commit f253a66382.

* Retain points by multiplier within each tile, not in global probability

* Test that intends to verify that the multiplier is reversible

* Get the test to detect the discrepancy

* Mark the start of multiplier clusters with a magic attribute

* Add string-contains

* Add in and ni operators

* Revert "Look up the attribute value for ordering"

This reverts commit 56bc73e49a.

* Revert "Type promotion for comparison"

This reverts commit 6f3256f5af.

* Make number formatting in tippecanoe_decisions consistent

* Revert "Add a test for the inverse spatial ordering"

This reverts commit c8047de9ab.

* Revert "Separate ordering by feature_minzoom from ordering inverse-spatially"

This reverts commit 35b19a223c.

* Revert "Fix inverse spatial feature order"

This reverts commit 5978ecdb44.

* Revert "Work in progress on inverse spatial ordering"

This reverts commit fdf230f632.

* Somehow missed the tests associated with that last revert

* Round-robin assign attributes to partials from across the multiplier

* Count the multiplier separately in each layer

* Fix distribution of accumulated attribute across multiplier features

* Update changelog, version, and docs

* Add "is null" and "isnt null" expressions

* Update interpretation of FSL expressions to pass the tests

* Test to assert that polygons are unaffected by the multiplier

* Clean up and comment

* Remove accidental unused case

CHANGELOG.md

@@ -1,3 +1,10 @@
+# 2.41.0
+
+* Add Felt-style expression support for -j feature filters
+* Add --retain-points-multiplier option
+* Add tippecanoe_decisions metadata field to record basezoom, drop rate, and multiplier
+* Add multiplier thinning (-m) and feature filters (-j) to tippecanoe-overzoom
+
 # 2.40.0
 
 * Slightly reduce compression aggressiveness to improve as-needed dropping speed

Makefile

@@ -76,7 +76,7 @@ tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
 unit: unit.o text.o sort.o mvt.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
-tippecanoe-overzoom: overzoom.o mvt.o clip.o
+tippecanoe-overzoom: overzoom.o mvt.o clip.o evaluator.o jsonpull/jsonpull.o text.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 -include $(wildcard *.d)
@@ -287,6 +287,37 @@ overzoom-test: tippecanoe-overzoom
 	./tippecanoe-overzoom -o tests/pbf/14-2616-6331.pbf tests/pbf/11-327-791.pbf 11/327/791 14/2616/6331
 	cmp tests/pbf/14-2616-6331.pbf /dev/null
 	rm tests/pbf/14-2616-6331.pbf
+	# Thinning
+	# 243 features in the source tile tests/pbf/0-0-0-pop.pbf
+	# 9 of them survive as the best of each cluster of 30
+	# ./tippecanoe -z1 -r30 --retain-points-multiplier 30 -f -e out.dir tests/ne_110m_populated_places/in.json
+	# cp out.dir/0/0/0.pbf tests/pbf/0-0-0-pop.pbf
+	./tippecanoe-overzoom -y NAME -m -o tests/pbf/0-0-0-pop-filtered.pbf tests/pbf/0-0-0-pop.pbf 0/0/0 0/0/0
+	./tippecanoe-decode tests/pbf/0-0-0-pop-filtered.pbf 0 0 0 > tests/pbf/0-0-0-pop-filtered.pbf.json.check
+	cmp tests/pbf/0-0-0-pop-filtered.pbf.json.check tests/pbf/0-0-0-pop-filtered.pbf.json
+	rm tests/pbf/0-0-0-pop-filtered.pbf tests/pbf/0-0-0-pop-filtered.pbf.json.check
+	# Filtering
+	# 243 features in the source tile tests/pbf/0-0-0-pop.pbf
+	# 27 of them match the filter and are retained
+	./tippecanoe-overzoom -y NAME -j'{"*":["SCALERANK","eq",0]}' -o tests/pbf/0-0-0-pop-expr.pbf tests/pbf/0-0-0-pop.pbf 0/0/0 0/0/0
+	./tippecanoe-decode tests/pbf/0-0-0-pop-expr.pbf 0 0 0 > tests/pbf/0-0-0-pop-expr.pbf.json.check
+	cmp tests/pbf/0-0-0-pop-expr.pbf.json.check tests/pbf/0-0-0-pop-expr.pbf.json
+	rm tests/pbf/0-0-0-pop-expr.pbf tests/pbf/0-0-0-pop-expr.pbf.json.check
+	# Filtering with multiplier
+	# 243 features in the source tile tests/pbf/0-0-0-pop.pbf
+	# 8 features survive into the output, from 9 clusters of 30
+	./tippecanoe-overzoom -y NAME -y SCALERANK -j'{"*":["SCALERANK","eq",0]}' -m -o tests/pbf/0-0-0-filter-mult.pbf tests/pbf/0-0-0-pop.pbf 0/0/0 0/0/0
+	./tippecanoe-decode tests/pbf/0-0-0-filter-mult.pbf 0 0 0 > tests/pbf/0-0-0-filter-mult.pbf.json.check
+	cmp tests/pbf/0-0-0-filter-mult.pbf.json.check tests/pbf/0-0-0-filter-mult.pbf.json
+	rm tests/pbf/0-0-0-filter-mult.pbf tests/pbf/0-0-0-filter-mult.pbf.json.check
+	# Test that overzooming with a multiplier exactly reverses the effect of tiling with a multiplier
+	./tippecanoe -q -z5 --preserve-point-density-threshold 8 --retain-points-multiplier 3 -f -e tests/muni/out/out.dir tests/muni/muni.json
+	./tippecanoe -q -z5 --preserve-point-density-threshold 8 -f -o tests/muni/out/out.mbtiles tests/muni/muni.json
+	./tippecanoe-overzoom -m -o tests/muni/out/out.dir/000.pbf tests/muni/out/out.dir/0/0/0.pbf 0/0/0 0/0/0
+	./tippecanoe-decode tests/muni/out/out.mbtiles 0 0 0 > tests/muni/out/out.dir/direct.json
+	./tippecanoe-decode tests/muni/out/out.dir/000.pbf 0 0 0 > tests/muni/out/out.dir/overzoomed.json
+	cmp tests/muni/out/out.dir/overzoomed.json tests/muni/out/out.dir/direct.json
+	rm -rf tests/muni/out/out.dir tests/muni/out/out.mbtiles tests/muni/out/out.dir/overzoomed.json tests/muni/out/out.dir/direct.json
 
 join-test: tippecanoe tippecanoe-decode tile-join
 	./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}'

README.md

@@ -421,7 +421,7 @@ be reduced to the maximum that can be used with the specified _maxzoom_.
 
 ### Filtering features by attributes
 
- * `-j` *filter* or `--feature-filter`=*filter*: Check features against a per-layer filter (as defined in the [Mapbox GL Style Specification](https://docs.mapbox.com/mapbox-gl-js/style-spec/#other-filter)) and only include those that match. Any features in layers that have no filter specified will be passed through. Filters for the layer `"*"` apply to all layers. The special variable `$zoom` refers to the current zoom level.
+ * `-j` *filter* or `--feature-filter`=*filter*: Check features against a per-layer filter (as defined in the [Mapbox GL Style Specification](https://docs.mapbox.com/mapbox-gl-js/style-spec/#other-filter) or in a Felt filter specification still to be finalized) and only include those that match. Any features in layers that have no filter specified will be passed through. Filters for the layer `"*"` apply to all layers. The special variable `$zoom` refers to the current zoom level.
  * `-J` *filter-file* or `--feature-filter-file`=*filter-file*: Like `-j`, but read the filter from a file.
 
 Example: to find the Natural Earth countries with low `scalerank` but high `LABELRANK`:
@@ -459,6 +459,7 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
    If you use `-Bg`, it will guess a zoom level that will keep at most 50,000 features in the densest tile.
    You can also specify a marker-width with `-Bg`*width* to allow fewer features in the densest tile to
    compensate for the larger marker, or `-Bf`*number* to allow at most *number* features in the densest tile.
+ * `--retain-points-multiplier=`_multiple_: Retain the specified multiple of points instead of just the number of points that would ordinarily be retained by the drop rate. These can be thinned out later with the `-m` option to `tippecanoe-overzoom`. The start of each cluster is marked in the feature sequence by the `tippecanoe:retain_points_multiplier_first` attribute.
  * `--drop-denser=`_percentage_: When dropping dots at zoom levels below the base zoom, give the specified _percentage_
    preference to retaining points in sparse areas and dropping points in dense areas.
  * `--limit-base-zoom-to-maximum-zoom` or `-Pb`: Limit the guessed base zoom not to exceed the maxzoom, even if this would put more than the requested number of features in a base zoom tile.
@@ -993,4 +994,6 @@ reads tile `inz/inx/iny` of `in.mvt.gz` and produces tile `outz/outx/outy` of `o
  * `-b` *buffer*: Set the tile buffer in the output tile (default 5)
  * `-d` *detail*: Set the detail of the output tile (default 12)
  * `-y` *attribute*: Retain the specified *attribute* in the output features. All attributes that are not named in a `-y` option will be removed.
+ * `-j` *filter*: Filter features using the same expression syntax as in tippecanoe.
+ * `-m`: If a tile was created with the `--retain-points-multiplier` option, thin the tile back down to its normal feature count during overzooming. The first feature from each cluster will be retained, unless `-j` is used to specify a filter, in which case the first matching filter from each cluster will be retained instead.
 

clip.cpp

@@ -7,6 +7,7 @@
 #include "errors.hpp"
 #include "compression.hpp"
 #include "mvt.hpp"
+#include "evaluator.hpp"
 
 static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<double, double>> &geom,
 							 long long minx, long long miny, long long maxx, long long maxy,
@@ -754,7 +755,8 @@ static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<d
 
 std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
-		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
+		     bool demultiply, json_object *filter) {
 	mvt_tile tile;
 
 	try {
@@ -768,12 +770,49 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
 		exit(EXIT_PROTOBUF);
 	}
 
-	return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles);
+	return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles, demultiply, filter);
+}
+
+struct tile_feature {
+	drawvec geom;
+	int t;
+	bool has_id;
+	unsigned long long id;
+	std::vector<unsigned> tags;
+	mvt_layer const *layer;
+};
+
+void feature_out(tile_feature const &feature, mvt_layer &outlayer, std::set<std::string> const &keep) {
+	// Add geometry to output feature
+
+	mvt_feature outfeature;
+	outfeature.type = feature.t;
+	for (auto const &g : feature.geom) {
+		outfeature.geometry.emplace_back(g.op, g.x, g.y);
+	}
+
+	// ID and attributes, if it didn't get clipped away
+
+	if (outfeature.geometry.size() > 0) {
+		if (feature.has_id) {
+			outfeature.has_id = true;
+			outfeature.id = feature.id;
+		}
+
+		for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
+			if (keep.size() == 0 || keep.find(feature.layer->keys[feature.tags[i]]) != keep.end()) {
+				outlayer.tag(outfeature, feature.layer->keys[feature.tags[i]], feature.layer->values[feature.tags[i + 1]]);
+			}
+		}
+
+		outlayer.features.push_back(outfeature);
+	}
 }
 
 std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
-		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
+		     bool demultiply, json_object *filter) {
 	mvt_tile outtile;
 
 	for (auto const &layer : tile.layers) {
@@ -788,8 +827,37 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
 		outlayer.version = layer.version;
 		outlayer.extent = 1LL << det;
 
-		for (auto const &feature : layer.features) {
+		std::vector<tile_feature> pending_tile_features;
-			mvt_feature outfeature;
+
+		for (auto feature : layer.features) {
+			bool flush_multiplier_cluster = false;
+			if (demultiply) {
+				for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
+					if (layer.keys[feature.tags[i]] == "tippecanoe:retain_points_multiplier_first") {
+						mvt_value v = layer.values[feature.tags[i + 1]];
+						if (v.type == mvt_bool && v.numeric_value.bool_value) {
+							flush_multiplier_cluster = true;
+							feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
+							break;
+						}
+					}
+				}
+			} else {
+				flush_multiplier_cluster = true;
+			}
+
+			if (flush_multiplier_cluster) {
+				if (pending_tile_features.size() > 0) {
+					feature_out(pending_tile_features[0], outlayer, keep);
+					pending_tile_features.clear();
+				}
+			}
+
+			std::set<std::string> exclude_attributes;
+			if (!evaluate(feature, layer, filter, exclude_attributes, nz)) {
+				continue;
+			}
+
 			drawvec geom;
 			int t = feature.type;
 
@@ -862,29 +930,20 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
 				geom = close_poly(geom);
 			}
 
-			// Add geometry to output feature
+			tile_feature tf;
+			tf.geom = geom;
+			tf.t = t;
+			tf.has_id = feature.has_id;
+			tf.id = feature.id;
+			tf.tags = feature.tags;
+			tf.layer = &layer;
 
-			outfeature.type = t;
+			pending_tile_features.push_back(tf);
-			for (auto const &g : geom) {
+		}
-				outfeature.geometry.emplace_back(g.op, g.x, g.y);
-			}
 
-			// ID and attributes, if it didn't get clipped away
+		if (pending_tile_features.size() > 0) {
-
+			feature_out(pending_tile_features[0], outlayer, keep);
-			if (outfeature.geometry.size() > 0) {
+			pending_tile_features.clear();
-				if (feature.has_id) {
-					outfeature.has_id = true;
-					outfeature.id = feature.id;
-				}
-
-				for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
-					if (keep.size() == 0 || keep.find(layer.keys[feature.tags[i]]) != keep.end()) {
-						outlayer.tag(outfeature, layer.keys[feature.tags[i]], layer.values[feature.tags[i + 1]]);
-					}
-				}
-
-				outlayer.features.push_back(outfeature);
-			}
 		}
 
 		if (outlayer.features.size() > 0) {
@@ -905,7 +964,8 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
 				for (size_t y = 0; y < 2; y++) {
 					std::string child = overzoom(outtile, nz, nx, ny,
 								     nz + 1, nx * 2 + x, ny * 2 + y,
-								     detail, buffer, keep, false, NULL);
+								     detail, buffer, keep, false, NULL,
+								     demultiply, filter);
 					if (child.size() > 0) {
 						next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y);
 					}

dirtiles.cpp

@@ -318,6 +318,9 @@ void dir_write_metadata(const char *outdir, const metadata &m) {
 		if (m.strategies_json.size() > 0) {
 			out(state, "strategies", m.strategies_json);
 		}
+		if (m.decisions_json.size() > 0) {
+			out(state, "tippecanoe_decisions", m.decisions_json);
+		}
 		out(state, "format", m.format);
 		out(state, "generator", m.generator);
 		out(state, "generator_options", m.generator_options);

evaluator.cpp

@@ -5,6 +5,123 @@
 #include "mvt.hpp"
 #include "evaluator.hpp"
 #include "errors.hpp"
+#include "milo/dtoa_milo.h"
+
+static std::string mvt_value_to_string(mvt_value one, bool &fail) {
+	if (one.type == mvt_string) {
+		return one.string_value;
+	} else if (one.type == mvt_float) {
+		return milo::dtoa_milo(one.numeric_value.float_value);
+	} else if (one.type == mvt_double) {
+		return milo::dtoa_milo(one.numeric_value.double_value);
+	} else if (one.type == mvt_int) {
+		return std::to_string(one.numeric_value.int_value);
+	} else if (one.type == mvt_uint) {
+		return std::to_string(one.numeric_value.uint_value);
+	} else if (one.type == mvt_sint) {
+		return std::to_string(one.numeric_value.sint_value);
+	} else if (one.type == mvt_bool) {
+		return one.numeric_value.bool_value ? "true" : "false";
+	} else if (one.type == mvt_null) {
+		fail = true;  // null op string => null
+		return "";
+	} else {
+		fprintf(stderr, "unhandled mvt_type %d\n", one.type);
+		exit(EXIT_IMPOSSIBLE);
+	}
+}
+
+int compare_fsl(mvt_value one, json_object *two, bool &fail) {
+	// In FSL expressions, the attribute value is coerced to the type
+	// of the JSON literal value it is being compared to.
+	//
+	// If it cannot be converted, the comparison returns null
+	// (which is distinct from true and false but is falsy if
+	// it is the final output value).
+
+	if (two->type == JSON_NULL) {
+		fail = true;  // anything op null => null
+		return 0;
+	}
+
+	if (two->type == JSON_NUMBER) {
+		double lhs;
+
+		if (one.type == mvt_string) {
+			char *endptr = NULL;
+			const char *s = one.string_value.c_str();
+			lhs = strtod(s, &endptr);
+			if (endptr == s) {
+				fail = true;  // non-numeric-string op number => null
+				return 0;
+			}
+		} else if (one.type == mvt_float) {
+			lhs = one.numeric_value.float_value;
+		} else if (one.type == mvt_double) {
+			lhs = one.numeric_value.double_value;
+		} else if (one.type == mvt_int) {
+			lhs = one.numeric_value.int_value;
+		} else if (one.type == mvt_uint) {
+			lhs = one.numeric_value.uint_value;
+		} else if (one.type == mvt_sint) {
+			lhs = one.numeric_value.sint_value;
+		} else if (one.type == mvt_bool) {
+			lhs = one.numeric_value.bool_value;
+		} else if (one.type == mvt_null) {
+			fail = true;  // null op number => null
+			return 0;
+		} else {
+			fprintf(stderr, "unhandled mvt_type %d\n", one.type);
+			exit(EXIT_IMPOSSIBLE);
+		}
+
+		if (lhs < two->value.number.number) {
+			return -1;
+		} else if (lhs > two->value.number.number) {
+			return 1;
+		} else {
+			return 0;
+		}
+	}
+
+	if (two->type == JSON_STRING) {
+		std::string lhs = mvt_value_to_string(one, fail);
+
+		return strcmp(lhs.c_str(), two->value.string.string);
+	}
+
+	if (two->type == JSON_TRUE || two->type == JSON_FALSE) {
+		bool lhs;
+
+		if (one.type == mvt_string) {
+			lhs = one.string_value.size() > 0;
+		} else if (one.type == mvt_float) {
+			lhs = one.numeric_value.float_value != 0;
+		} else if (one.type == mvt_double) {
+			lhs = one.numeric_value.double_value != 0;
+		} else if (one.type == mvt_int) {
+			lhs = one.numeric_value.int_value != 0;
+		} else if (one.type == mvt_uint) {
+			lhs = one.numeric_value.uint_value != 0;
+		} else if (one.type == mvt_sint) {
+			lhs = one.numeric_value.sint_value != 0;
+		} else if (one.type == mvt_bool) {
+			lhs = one.numeric_value.bool_value;
+		} else if (one.type == mvt_null) {
+			fail = true;  // null op bool => null
+			return 0;
+		} else {
+			fprintf(stderr, "unhandled mvt_type %d\n", one.type);
+			exit(EXIT_IMPOSSIBLE);
+		}
+
+		bool rhs = two->type == JSON_TRUE;
+		return lhs - rhs;
+	}
+
+	fprintf(stderr, "unhandled JSON type %d\n", two->type);
+	exit(EXIT_IMPOSSIBLE);
+}
 
 int compare(mvt_value one, json_object *two, bool &fail) {
 	if (one.type == mvt_string) {
@@ -70,7 +187,36 @@ int compare(mvt_value one, json_object *two, bool &fail) {
 	exit(EXIT_IMPOSSIBLE);
 }
 
-bool eval(std::map<std::string, mvt_value> const &feature, json_object *f, std::set<std::string> &exclude_attributes) {
+// 0: false
+// 1: true
+// -1: incomparable (sql null), treated as false in final output
+static int eval(std::map<std::string, mvt_value> const &feature, json_object *f, std::set<std::string> &exclude_attributes) {
+	if (f != NULL) {
+		if (f->type == JSON_TRUE) {
+			return 1;
+		} else if (f->type == JSON_FALSE) {
+			return 0;
+		} else if (f->type == JSON_NULL) {
+			return -1;
+		}
+
+		if (f->type == JSON_NUMBER) {
+			if (f->value.number.number == 0) {
+				return 0;
+			} else {
+				return 1;
+			}
+		}
+
+		if (f->type == JSON_STRING) {
+			if (f->value.string.string[0] == '\0') {
+				return 0;
+			} else {
+				return 1;
+			}
+		}
+	}
+
 	if (f == NULL || f->type != JSON_ARRAY) {
 		fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f));
 		exit(EXIT_FILTER);
@@ -81,6 +227,149 @@ bool eval(std::map<std::string, mvt_value> const &feature, json_object *f, std::
 		exit(EXIT_FILTER);
 	}
 
+	// FSL conjunctions
+	if (f->value.array.length == 3 &&
+	    f->value.array.array[1]->type == JSON_STRING &&
+	    (strcmp(f->value.array.array[1]->value.string.string, "or") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "and") == 0)) {
+		int lhs;
+
+		if (f->value.array.array[0]->type == JSON_STRING) {
+			// if LHS of a boolean conjunction is a string, it is an attribute reference
+			auto ff = feature.find(std::string(f->value.array.array[0]->value.string.string));
+			if (ff != feature.end()) {
+				if (ff->second.type == mvt_bool) {
+					lhs = ff->second.numeric_value.bool_value;
+				} else {
+					lhs = -1;  // not boolean: null
+				}
+			} else {
+				lhs = -1;  // not found: null
+			}
+		} else {
+			lhs = eval(feature, f->value.array.array[0], exclude_attributes);
+		}
+
+		int rhs = eval(feature, f->value.array.array[2], exclude_attributes);
+		if (lhs < 0 && rhs < 0) {
+			return -1;  // null op null => null
+		}
+		if (strcmp(f->value.array.array[1]->value.string.string, "or") == 0) {
+			return (lhs > 0) || (rhs > 0);
+		} else {
+			return (lhs > 0) && (rhs > 0);
+		}
+	}
+
+	// FSL comparators
+	if (f->value.array.length == 3 &&
+	    f->value.array.array[0]->type == JSON_STRING &&
+	    f->value.array.array[1]->type == JSON_STRING &&
+	    (strcmp(f->value.array.array[1]->value.string.string, "lt") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "gt") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "le") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "ge") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "eq") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "ne") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "cn") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "nc") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "in") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "ni") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "is") == 0 ||
+	     strcmp(f->value.array.array[1]->value.string.string, "isnt") == 0 ||
+	     false)) {
+		mvt_value lhs;
+		lhs.type = mvt_null;  // attributes that aren't found are nulls
+		auto ff = feature.find(std::string(f->value.array.array[0]->value.string.string));
+		if (ff != feature.end()) {
+			lhs = ff->second;
+		}
+
+		if (f->value.array.array[2]->type == JSON_NULL && strcmp(f->value.array.array[1]->value.string.string, "is") == 0) {
+			return lhs.type == mvt_null;  // null is null => true, anything is null => false
+		}
+		if (f->value.array.array[2]->type == JSON_NULL && strcmp(f->value.array.array[1]->value.string.string, "isnt") == 0) {
+			return lhs.type != mvt_null;  // null isnt null => false, anything isnt null => true
+		}
+
+		if (lhs.type == mvt_null) {
+			return -1;  // null op anything => null
+		}
+
+		bool fail = false;
+
+		if (f->value.array.array[2]->type == JSON_STRING &&
+		    (strcmp(f->value.array.array[1]->value.string.string, "cn") == 0 ||
+		     strcmp(f->value.array.array[1]->value.string.string, "nc") == 0)) {
+			std::string s = mvt_value_to_string(lhs, fail);
+			if (fail) {
+				return -1;  // null cn anything => false
+			}
+
+			bool contains = strstr(s.c_str(), f->value.array.array[2]->value.string.string);
+			if (strcmp(f->value.array.array[1]->value.string.string, "cn") == 0) {
+				return contains;
+			} else {
+				return !contains;
+			}
+		}
+
+		if (f->value.array.array[2]->type == JSON_ARRAY &&
+		    (strcmp(f->value.array.array[1]->value.string.string, "in") == 0 ||
+		     strcmp(f->value.array.array[1]->value.string.string, "ni") == 0)) {
+			std::string s = mvt_value_to_string(lhs, fail);
+			if (fail) {
+				return -1;  // null in anything => false
+			}
+
+			bool contains = false;
+			for (size_t i = 0; i < f->value.array.array[2]->value.array.length; i++) {
+				if (f->value.array.array[2]->value.array.array[i]->type != JSON_STRING) {
+					return -1;  // anything in [not-a-string] => null
+				}
+
+				if (s == f->value.array.array[2]->value.array.array[i]->value.string.string) {
+					contains = true;
+					break;
+				}
+			}
+
+			if (strcmp(f->value.array.array[1]->value.string.string, "in") == 0) {
+				return contains;
+			} else {
+				return !contains;
+			}
+		}
+
+		int cmp = compare_fsl(ff->second, f->value.array.array[2], fail);
+		if (fail) {
+			printf("cast fail\n");
+			return -1;  // null
+		}
+
+		if (strcmp(f->value.array.array[1]->value.string.string, "eq") == 0) {
+			return cmp == 0;
+		}
+		if (strcmp(f->value.array.array[1]->value.string.string, "ne") == 0) {
+			return cmp != 0;
+		}
+		if (strcmp(f->value.array.array[1]->value.string.string, "gt") == 0) {
+			return cmp > 0;
+		}
+		if (strcmp(f->value.array.array[1]->value.string.string, "ge") == 0) {
+			return cmp >= 0;
+		}
+		if (strcmp(f->value.array.array[1]->value.string.string, "lt") == 0) {
+			return cmp < 0;
+		}
+		if (strcmp(f->value.array.array[1]->value.string.string, "le") == 0) {
+			return cmp <= 0;
+		}
+
+		fprintf(stderr, "expression fsl comparison: can't happen %s\n", f->value.array.array[1]->value.string.string);
+		exit(EXIT_IMPOSSIBLE);
+	}
+
 	if (f->value.array.array[0]->type != JSON_STRING) {
 		fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f));
 		exit(EXIT_FILTER);
@@ -192,17 +481,19 @@ bool eval(std::map<std::string, mvt_value> const &feature, json_object *f, std::
 		}
 
 		for (size_t i = 1; i < f->value.array.length; i++) {
-			bool out = eval(feature, f->value.array.array[i], exclude_attributes);
+			int out = eval(feature, f->value.array.array[i], exclude_attributes);
 
-			if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) {
+			if (out >= 0) {	 // nulls are ignored in boolean and/or expressions
-				v = v && out;
+				if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) {
-				if (!v) {
+					v = v && out;
-					break;
+					if (!v) {
-				}
+						break;
-			} else {
+					}
-				v = v || out;
+				} else {
-				if (v) {
+					v = v || out;
-					break;
+					if (v) {
+						break;
+					}
 				}
 			}
 		}
@@ -281,7 +572,7 @@ bool eval(std::map<std::string, mvt_value> const &feature, json_object *f, std::
 			exit(EXIT_FILTER);
 		}
 
-		bool ok = eval(feature, f->value.array.array[2], exclude_attributes);
+		bool ok = eval(feature, f->value.array.array[2], exclude_attributes) > 0;
 		if (!ok) {
 			exclude_attributes.insert(f->value.array.array[1]->value.string.string);
 		}
@@ -304,12 +595,12 @@ bool evaluate(std::map<std::string, mvt_value> const &feature, std::string const
 
 	f = json_hash_get(filter, layer.c_str());
 	if (ok && f != NULL) {
-		ok = eval(feature, f, exclude_attributes);
+		ok = eval(feature, f, exclude_attributes) > 0;
 	}
 
 	f = json_hash_get(filter, "*");
 	if (ok && f != NULL) {
-		ok = eval(feature, f, exclude_attributes);
+		ok = eval(feature, f, exclude_attributes) > 0;
 	}
 
 	return ok;
@@ -346,3 +637,49 @@ json_object *parse_filter(const char *s) {
 	json_end(jp);
 	return filter;
 }
+
+bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set<std::string> &exclude_attributes, int z) {
+	if (filter != NULL) {
+		std::map<std::string, mvt_value> attributes;
+
+		for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) {
+			std::string key = layer.keys[feat.tags[t]];
+			const mvt_value &val = layer.values[feat.tags[t + 1]];
+
+			attributes.insert(std::pair<std::string, mvt_value>(key, val));
+		}
+
+		if (feat.has_id) {
+			mvt_value v;
+			v.type = mvt_uint;
+			v.numeric_value.uint_value = feat.id;
+
+			attributes.insert(std::pair<std::string, mvt_value>("$id", v));
+		}
+
+		mvt_value v;
+		v.type = mvt_string;
+
+		if (feat.type == mvt_point) {
+			v.string_value = "Point";
+		} else if (feat.type == mvt_linestring) {
+			v.string_value = "LineString";
+		} else if (feat.type == mvt_polygon) {
+			v.string_value = "Polygon";
+		}
+
+		attributes.insert(std::pair<std::string, mvt_value>("$type", v));
+
+		mvt_value v2;
+		v2.type = mvt_uint;
+		v2.numeric_value.uint_value = z;
+
+		attributes.insert(std::pair<std::string, mvt_value>("$zoom", v2));
+
+		if (!evaluate(attributes, layer.name, filter, exclude_attributes)) {
+			return false;
+		}
+	}
+
+	return true;
+}

evaluator.hpp

@@ -11,4 +11,6 @@ bool evaluate(std::map<std::string, mvt_value> const &feature, std::string const
 json_object *parse_filter(const char *s);
 json_object *read_filter(const char *fname);
 
+bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set<std::string> &exclude_attributes, int z);
+
 #endif

geometry.hpp

@@ -8,6 +8,7 @@
 #include <sqlite3.h>
 #include <stdio.h>
 #include <mvt.hpp>
+#include "jsonpull/jsonpull.h"
 
 #define VT_POINT 1
 #define VT_LINE 2
@@ -100,10 +101,12 @@ double distance_from_line(long long point_x, long long point_y, long long segA_x
 
 std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
-		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
+		     bool demultiply, json_object *filter);
 
 std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
-		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
+		     bool demultiply, json_object *filter);
 
 #endif

main.cpp

@@ -93,6 +93,7 @@ unsigned int drop_denser = 0;
 std::map<std::string, serial_val> set_attributes;
 unsigned long long preserve_point_density_threshold = 0;
 long long extend_zooms_max = 0;
+int retain_points_multiplier = 1;
 
 std::vector<order_field> order_by;
 bool order_reverse;
@@ -303,7 +304,6 @@ int calc_feature_minzoom(struct index *ix, struct drop_state *ds, int maxzoom, d
 		for (ssize_t i = maxzoom; i >= 0; i--) {
 			ds[i].seq++;
 		}
-		ssize_t chosen = maxzoom + 1;
 		for (ssize_t i = maxzoom; i >= 0; i--) {
 			if (ds[i].seq < 0) {
 				feature_minzoom = i + 1;
@@ -317,7 +317,6 @@ int calc_feature_minzoom(struct index *ix, struct drop_state *ds, int maxzoom, d
 					ds[j].previndex = ix->ix;
 				}
 
-				chosen = i + 1;
 				break;
 			} else {
 				ds[i].seq -= ds[i].interval;
@@ -330,7 +329,7 @@ int calc_feature_minzoom(struct index *ix, struct drop_state *ds, int maxzoom, d
 		// we will go ahead and push it out.
 
 		if (preserve_point_density_threshold > 0) {
-			for (ssize_t i = 0; i < chosen && i < maxzoom; i++) {
+			for (ssize_t i = 0; i < feature_minzoom && i < maxzoom; i++) {
 				if (ix->ix - ds[i].previndex > ((1LL << (32 - i)) / preserve_point_density_threshold) * ((1LL << (32 - i)) / preserve_point_density_threshold)) {
 					feature_minzoom = i;
 
@@ -2825,7 +2824,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 		ai->second.maxzoom = maxzoom;
 	}
 
-	metadata m = make_metadata(fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, minlat2, minlon2, maxlat2, maxlon2, midlat, midlon, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies);
+	metadata m = make_metadata(fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, minlat2, minlon2, maxlat2, maxlon2, midlat, midlon, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies, basezoom, droprate, retain_points_multiplier);
 	if (outdb != NULL) {
 		mbtiles_write_metadata(outdb, m, forcetable);
 	} else {
@@ -3150,6 +3149,7 @@ int main(int argc, char **argv) {
 
 		{"Dropping a fixed fraction of features by zoom level", 0, 0, 0},
 		{"drop-rate", required_argument, 0, 'r'},
+		{"retain-points-multiplier", required_argument, 0, '~'},
 		{"base-zoom", required_argument, 0, 'B'},
 		{"drop-denser", required_argument, 0, '~'},
 		{"limit-base-zoom-to-maximum-zoom", no_argument, &prevent[P_BASEZOOM_ABOVE_MAXZOOM], 1},
@@ -3370,6 +3370,8 @@ int main(int argc, char **argv) {
 				preserve_point_density_threshold = atoll_require(optarg, "Preserve point density threshold");
 			} else if (strcmp(opt, "extend-zooms-if-still-dropping-maximum") == 0) {
 				extend_zooms_max = atoll_require(optarg, "Maximum number by which to extend zooms");
+			} else if (strcmp(opt, "retain-points-multiplier") == 0) {
+				retain_points_multiplier = atoll_require(optarg, "Multiply the fraction of points retained by zoom level");
 			} else {
 				fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt);
 				exit(EXIT_ARGS);

main.hpp

@@ -62,6 +62,7 @@ extern size_t limit_tile_feature_count;
 extern size_t limit_tile_feature_count_at_maxzoom;
 extern std::map<std::string, serial_val> set_attributes;
 extern long long extend_zooms_max;
+extern int retain_points_multiplier;
 
 struct order_field {
 	std::string name;

man/tippecanoe.1

@@ -526,7 +526,7 @@ The attributes and values may also be specified as JSON keys and values: \fB\fC\
 .SS Filtering features by attributes
 .RS
 .IP \(bu 2
-\fB\fC\-j\fR \fIfilter\fP or \fB\fC\-\-feature\-filter\fR=\fIfilter\fP: Check features against a per\-layer filter (as defined in the Mapbox GL Style Specification \[la]https://docs.mapbox.com/mapbox-gl-js/style-spec/#other-filter\[ra]) and only include those that match. Any features in layers that have no filter specified will be passed through. Filters for the layer \fB\fC"*"\fR apply to all layers. The special variable \fB\fC$zoom\fR refers to the current zoom level.
+\fB\fC\-j\fR \fIfilter\fP or \fB\fC\-\-feature\-filter\fR=\fIfilter\fP: Check features against a per\-layer filter (as defined in the Mapbox GL Style Specification \[la]https://docs.mapbox.com/mapbox-gl-js/style-spec/#other-filter\[ra] or in a Felt filter specification still to be finalized) and only include those that match. Any features in layers that have no filter specified will be passed through. Filters for the layer \fB\fC"*"\fR apply to all layers. The special variable \fB\fC$zoom\fR refers to the current zoom level.
 .IP \(bu 2
 \fB\fC\-J\fR \fIfilter\-file\fP or \fB\fC\-\-feature\-filter\-file\fR=\fIfilter\-file\fP: Like \fB\fC\-j\fR, but read the filter from a file.
 .RE
@@ -574,6 +574,8 @@ If you use \fB\fC\-Bg\fR, it will guess a zoom level that will keep at most 50,0
 You can also specify a marker\-width with \fB\fC\-Bg\fR\fIwidth\fP to allow fewer features in the densest tile to
 compensate for the larger marker, or \fB\fC\-Bf\fR\fInumber\fP to allow at most \fInumber\fP features in the densest tile.
 .IP \(bu 2
+\fB\fC\-\-retain\-points\-multiplier=\fR\fImultiple\fP: Retain the specified multiple of points instead of just the number of points that would ordinarily be retained by the drop rate. These can be thinned out later with the \fB\fC\-m\fR option to \fB\fCtippecanoe\-overzoom\fR\&. The start of each cluster is marked in the feature sequence by the \fB\fCtippecanoe:retain_points_multiplier_first\fR attribute.
+.IP \(bu 2
 \fB\fC\-\-drop\-denser=\fR\fIpercentage\fP: When dropping dots at zoom levels below the base zoom, give the specified \fIpercentage\fP
 preference to retaining points in sparse areas and dropping points in dense areas.
 .IP \(bu 2
@@ -1260,4 +1262,8 @@ reads tile \fB\fCinz/inx/iny\fR of \fB\fCin.mvt.gz\fR and produces tile \fB\fCou
 \fB\fC\-d\fR \fIdetail\fP: Set the detail of the output tile (default 12)
 .IP \(bu 2
 \fB\fC\-y\fR \fIattribute\fP: Retain the specified \fIattribute\fP in the output features. All attributes that are not named in a \fB\fC\-y\fR option will be removed.
+.IP \(bu 2
+\fB\fC\-j\fR \fIfilter\fP: Filter features using the same expression syntax as in tippecanoe.
+.IP \(bu 2
+\fB\fC\-m\fR: If a tile was created with the \fB\fC\-\-retain\-points\-multiplier\fR option, thin the tile back down to its normal feature count during overzooming. The first feature from each cluster will be retained, unless \fB\fC\-j\fR is used to specify a filter, in which case the first matching filter from each cluster will be retained instead.
 .RE

mbtiles.cpp

@@ -7,6 +7,8 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sqlite3.h>
+#include <cmath>
+#include <climits>
 #include <vector>
 #include <string>
 #include <set>
@@ -601,6 +603,17 @@ void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable) {
 		sqlite3_free(sql);
 	}
 
+	if (m.decisions_json.size() > 0) {
+		sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", m.decisions_json.c_str());
+		if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
+			fprintf(stderr, "set decisions: %s\n", err);
+			if (!forcetable) {
+				exit(EXIT_SQLITE);
+			}
+		}
+		sqlite3_free(sql);
+	}
+
 	if (m.vector_layers_json.size() > 0 || m.tilestats_json.size() > 0) {
 		std::string json;
 		json_writer state(&json);
@@ -642,7 +655,7 @@ void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable) {
 	}
 }
 
-metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies) {
+metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies, int basezoom, double droprate, int retain_points_multiplier) {
 	metadata m;
 
 	m.name = fname;
@@ -677,6 +690,17 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
 
 	m.strategies_json = stringify_strategies(strategies);
 
+	if (isinf(droprate)) {
+		droprate = LLONG_MAX;
+	}
+	if (basezoom != maxzoom || droprate != 2.5 || retain_points_multiplier != 1) {
+		m.decisions_json = std::string("{") +
+				   "\"basezoom\":" + milo::dtoa_milo(basezoom) + "," +
+				   "\"droprate\":" + milo::dtoa_milo(droprate) + "," +
+				   "\"retain_points_multiplier\":" + std::to_string(retain_points_multiplier) +
+				   std::string("}");
+	}
+
 	if (vector) {
 		{
 			json_writer state(&m.vector_layers_json);

mbtiles.hpp

@@ -66,6 +66,7 @@ struct metadata {
 
 	std::string vector_layers_json;
 	std::string tilestats_json;
+	std::string decisions_json;
 };
 
 #include "tile.hpp"
@@ -75,7 +76,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
 void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
 void mbtiles_erase_zoom(sqlite3 *outdb, int z);
 
-metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies);
+metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies, int basezoom, double droprate, int retain_points_multiplier);
 void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable);
 
 void mbtiles_close(sqlite3 *outdb, const char *pgm);

overzoom.cpp

@@ -6,12 +6,15 @@
 #include "errors.hpp"
 #include "mvt.hpp"
 #include "geometry.hpp"
+#include "evaluator.hpp"
 
 extern char *optarg;
 extern int optind;
 
 int detail = 12;  // tippecanoe-style: mvt extent == 1 << detail
 int buffer = 5;	  // tippecanoe-style: mvt buffer == extent * buffer / 256;
+bool demultiply = false;
+std::string filter;
 
 std::set<std::string> keep;
 
@@ -25,7 +28,7 @@ int main(int argc, char **argv) {
 	int i;
 	const char *outfile = NULL;
 
-	while ((i = getopt(argc, argv, "y:o:d:b:")) != -1) {
+	while ((i = getopt(argc, argv, "y:d:b:o:mj:")) != -1) {
 		switch (i) {
 		case 'y':
 			keep.insert(optarg);
@@ -43,7 +46,16 @@ int main(int argc, char **argv) {
 			buffer = atoi(optarg);
 			break;
 
+		case 'm':
+			demultiply = true;
+			break;
+
+		case 'j':
+			filter = optarg;
+			break;
+
 		default:
+			fprintf(stderr, "Unrecognized flag -%c\n", i);
 			usage(argv);
 		}
 	}
@@ -91,7 +103,12 @@ int main(int argc, char **argv) {
 		exit(EXIT_FAILURE);
 	}
 
-	std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, true, NULL);
+	json_object *json_filter = NULL;
+	if (filter.size() > 0) {
+		json_filter = parse_filter(filter.c_str());
+	}
+
+	std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, true, NULL, demultiply, json_filter);
 	fwrite(out.c_str(), sizeof(char), out.size(), f);
 	fclose(f);
 

pmtiles_file.cpp

@@ -103,6 +103,13 @@ std::string metadata_to_pmtiles_json(metadata m) {
 		state.json_write_string("strategies");
 		state.json_write_json(m.strategies_json);
 	}
+
+	if (m.decisions_json.size() > 0) {
+		state.json_comma_newline();
+		state.json_write_string("tippecanoe_decisions");
+		state.json_write_json(m.decisions_json);
+	}
+
 	out(state, "generator", m.generator);
 	out(state, "generator_options", m.generator_options);
 
@@ -428,6 +435,12 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) {
 				fprintf(stderr, "set %s in metadata: %s\n", key, err);
 			}
 			sqlite3_free(sql);
+		} else if (strcmp(key, "tippecanoe_decisions") == 0 && o->value.object.values[i]->type == JSON_HASH) {
+			sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(o->value.object.values[i]));
+			if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
+				fprintf(stderr, "set %s in metadata: %s\n", key, err);
+			}
+			sqlite3_free(sql);
 		} else if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) {
 			fprintf(stderr, "%s\n", key);
 			fprintf(stderr, "%s: non-string in metadata\n", fname);

tests/accumulate/out/-z3_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma_-r1_-K100.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/accumulate/out/-z3_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma_-r1_-K100.json.check.mbtiles",
 "strategies": "[{\"coalesced_as_needed\":78},{\"coalesced_as_needed\":65},{\"coalesced_as_needed\":43},{\"coalesced_as_needed\":17}]",
+"tippecanoe_decisions": "{\"basezoom\":3,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/layer-json/out.mbtiles.json

@@ -8,6 +8,7 @@
 "maxzoom": "0",
 "minzoom": "0",
 "name": "tests/layer-json/out.mbtiles",
+"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/muni/out/-Z11_-z13_-B15.json

@@ -10,6 +10,7 @@
 "minzoom": "11",
 "name": "tests/muni/out/-Z11_-z13_-B15.json.check.mbtiles",
 "strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":4727},{\"dropped_by_rate\":4643},{\"dropped_by_rate\":4210}]",
+"tippecanoe_decisions": "{\"basezoom\":15,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/muni/out/-Z11_-z13_-Bf2000.json

@@ -10,6 +10,7 @@
 "minzoom": "11",
 "name": "tests/muni/out/-Z11_-z13_-Bf2000.json.check.mbtiles",
 "strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":2910},{},{}]",
+"tippecanoe_decisions": "{\"basezoom\":12,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/muni/out/-Z11_-z13_-rf2000.json

@@ -10,6 +10,7 @@
 "minzoom": "11",
 "name": "tests/muni/out/-Z11_-z13_-rf2000.json.check.mbtiles",
 "strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":2542},{\"dropped_by_rate\":1547},{}]",
+"tippecanoe_decisions": "{\"basezoom\":13,\"droprate\":1.4510341140028376,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/muni/out/-Z11_-z13_-rf2000_-Bg.json

@@ -10,6 +10,7 @@
 "minzoom": "11",
 "name": "tests/muni/out/-Z11_-z13_-rf2000_-Bg.json.check.mbtiles",
 "strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":2546},{},{}]",
+"tippecanoe_decisions": "{\"basezoom\":12,\"droprate\":2.1055,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/muni/out/-Z11_-z13_-rf2000_-g2.json

@@ -10,6 +10,7 @@
 "minzoom": "11",
 "name": "tests/muni/out/-Z11_-z13_-rf2000_-g2.json.check.mbtiles",
 "strategies": "[{},{},{},{},{},{},{},{},{},{},{},{\"dropped_by_rate\":1349,\"dropped_by_gamma\":963},{\"dropped_by_rate\":747,\"dropped_by_gamma\":752},{\"dropped_by_gamma\":514}]",
+"tippecanoe_decisions": "{\"basezoom\":13,\"droprate\":1.176222768016331,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/muni/out/-r1_-K20.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/muni/out/-r1_-K20.json.check.mbtiles",
 "strategies": "[{\"coalesced_as_needed\":4608},{\"coalesced_as_needed\":4608},{\"coalesced_as_needed\":4608},{\"coalesced_as_needed\":4608},{\"coalesced_as_needed\":4608},{\"coalesced_as_needed\":4607},{\"coalesced_as_needed\":4607},{\"coalesced_as_needed\":4606},{\"coalesced_as_needed\":6005},{\"coalesced_as_needed\":5393},{\"coalesced_as_needed\":5012},{\"coalesced_as_needed\":4727},{\"coalesced_as_needed\":4577},{\"coalesced_as_needed\":4003},{\"coalesced_as_needed\":3005}]",
+"tippecanoe_decisions": "{\"basezoom\":14,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--drop-densest-as-needed.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--drop-densest-as-needed.json.check.mbtiles",
 "strategies": "[{\"dropped_as_needed\":167,\"tile_size_desired\":39241},{\"dropped_as_needed\":184,\"tile_size_desired\":25163},{\"dropped_as_needed\":160,\"tiny_polygons\":1,\"tile_size_desired\":21214},{\"dropped_as_needed\":133,\"tile_size_desired\":10758},{\"dropped_as_needed\":58,\"tile_size_desired\":6601},{\"tiny_polygons\":2}]",
+"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--drop-fraction-as-needed.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--drop-fraction-as-needed.json.check.mbtiles",
 "strategies": "[{\"dropped_as_needed\":164,\"tile_size_desired\":39241},{\"dropped_as_needed\":187,\"tile_size_desired\":25163},{\"dropped_as_needed\":190,\"tiny_polygons\":1,\"tile_size_desired\":21214},{\"dropped_as_needed\":181,\"tile_size_desired\":10758},{\"dropped_as_needed\":147,\"tile_size_desired\":6601},{\"tiny_polygons\":2}]",
+"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--drop-smallest-as-needed.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--drop-smallest-as-needed.json.check.mbtiles",
 "strategies": "[{\"dropped_as_needed\":174,\"tile_size_desired\":39241},{\"dropped_as_needed\":200,\"tile_size_desired\":25163},{\"dropped_as_needed\":187,\"tiny_polygons\":1,\"tile_size_desired\":21214},{\"dropped_as_needed\":154,\"tile_size_desired\":10758},{\"dropped_as_needed\":79,\"tile_size_desired\":6601},{\"tiny_polygons\":2}]",
+"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--force-feature-limit.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_admin_0_countries/out/-ae_-zg_-M5000_--force-feature-limit.json.check.mbtiles",
 "strategies": "[{\"dropped_as_needed\":479,\"tile_size_desired\":39241},{\"dropped_as_needed\":264,\"tiny_polygons\":2,\"tile_size_desired\":25163},{\"dropped_as_needed\":221,\"tiny_polygons\":2,\"tile_size_desired\":21214},{\"dropped_as_needed\":93,\"tile_size_desired\":10758},{\"dropped_as_needed\":9,\"tile_size_desired\":6601},{\"tiny_polygons\":2}]",
+"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/--smallest-maximum-zoom-guess_2_-Bf20_-rf20_-pb.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/--smallest-maximum-zoom-guess_2_-Bf20_-rf20_-pb.json.check.mbtiles",
 "strategies": "[{\"dropped_by_rate\":237},{\"dropped_by_rate\":229},{}]",
+"tippecanoe_decisions": "{\"basezoom\":2,\"droprate\":6.599999999999999,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/--smallest-maximum-zoom-guess_3_-Bg.json

@@ -9,6 +9,7 @@
 "maxzoom": "3",
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/--smallest-maximum-zoom-guess_3_-Bg.json.check.mbtiles",
+"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/--smallest-maximum-zoom-guess_3_-rp.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/--smallest-maximum-zoom-guess_3_-rp.json.check.mbtiles",
 "strategies": "[{\"dropped_by_rate\":230},{\"dropped_by_rate\":231},{\"dropped_by_rate\":160},{}]",
+"tippecanoe_decisions": "{\"basezoom\":3,\"droprate\":2.6582825467008166,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-r1_-z8_-J_tests%2ffeature-filter%2fplaces-filter.json

@@ -9,6 +9,7 @@
 "maxzoom": "8",
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-r1_-z8_-J_tests%2ffeature-filter%2fplaces-filter.json.check.mbtiles",
+"tippecanoe_decisions": "{\"basezoom\":8,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-yNAME_-z5_--preserve-point-density-threshold_8.json

@@ -17,7 +17,7 @@
 { "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
 { "type": "Feature", "properties": { "NAME": "Vancouver" }, "geometry": { "type": "Point", "coordinates": [ -123.134766, 49.267805 ] } }
 ,
-{ "type": "Feature", "properties": { "NAME": "Denver" }, "geometry": { "type": "Point", "coordinates": [ -104.941406, 39.707187 ] } }
+{ "type": "Feature", "properties": { "NAME": "San Francisco" }, "geometry": { "type": "Point", "coordinates": [ -122.431641, 37.788081 ] } }
 ,
 { "type": "Feature", "properties": { "NAME": "Chicago" }, "geometry": { "type": "Point", "coordinates": [ -87.714844, 41.836828 ] } }
 ,

tests/ne_110m_populated_places/out/-yNAME_-z5_-B3.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-yNAME_-z5_-B3.json.check.mbtiles",
 "strategies": "[{\"dropped_by_rate\":227},{\"dropped_by_rate\":227},{\"dropped_by_rate\":152},{},{},{}]",
+"tippecanoe_decisions": "{\"basezoom\":3,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-yNAME_-z5_-r1.5.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-yNAME_-z5_-r1.5.json.check.mbtiles",
 "strategies": "[{\"dropped_by_rate\":211},{\"dropped_by_rate\":214},{\"dropped_by_rate\":179},{\"dropped_by_rate\":138},{\"dropped_by_rate\":70},{}]",
+"tippecanoe_decisions": "{\"basezoom\":5,\"droprate\":1.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-z0_-M10000_--drop-densest-as-needed_--extend-zooms-if-still-dropping-maximum_2.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-z0_-M10000_--drop-densest-as-needed_--extend-zooms-if-still-dropping-maximum_2.json.check.mbtiles",
 "strategies": "[{\"dropped_as_needed\":219,\"tile_size_desired\":74734},{\"dropped_as_needed\":207,\"tile_size_desired\":44415},{\"dropped_as_needed\":181,\"tile_size_desired\":35130}]",
+"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-z0_-r1_-yNAME_-ySOV0NAME_-yELEVATION_-YNAME%3aCity_-YSOV0NAME%3aCountry.json

@@ -9,6 +9,7 @@
 "maxzoom": "0",
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-z0_-r1_-yNAME_-ySOV0NAME_-yELEVATION_-YNAME%3aCity_-YSOV0NAME%3aCountry.json.check.mbtiles",
+"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-z3_-K20_-r1_-yNAME_-k2.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-z3_-K20_-r1_-yNAME_-k2.json.check.mbtiles",
 "strategies": "[{\"coalesced_as_needed\":207},{\"coalesced_as_needed\":174},{\"coalesced_as_needed\":108},{}]",
+"tippecanoe_decisions": "{\"basezoom\":3,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-z3_-r1_--limit-tile-feature-count_3.json

@@ -9,6 +9,7 @@
 "maxzoom": "3",
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-z3_-r1_--limit-tile-feature-count_3.json.check.mbtiles",
+"tippecanoe_decisions": "{\"basezoom\":3,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-z3_-r1_--limit-tile-feature-count_3_--limit-tile-feature-count-at-maximum-zoom_10.json

@@ -9,6 +9,7 @@
 "maxzoom": "3",
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-z3_-r1_--limit-tile-feature-count_3_--limit-tile-feature-count-at-maximum-zoom_10.json.check.mbtiles",
+"tippecanoe_decisions": "{\"basezoom\":3,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-zg_-K20_-r1_-yNAME.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-zg_-K20_-r1_-yNAME.json.check.mbtiles",
 "strategies": "[{\"coalesced_as_needed\":207},{\"coalesced_as_needed\":174},{\"coalesced_as_needed\":108},{\"coalesced_as_needed\":51}]",
+"tippecanoe_decisions": "{\"basezoom\":3,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/ne_110m_populated_places/out/-zg_-K20_-r1_-yNAME_-kg.json

@@ -10,6 +10,7 @@
 "minzoom": "0",
 "name": "tests/ne_110m_populated_places/out/-zg_-K20_-r1_-yNAME_-kg.json.check.mbtiles",
 "strategies": "[{\"coalesced_as_needed\":207},{\"coalesced_as_needed\":174},{\"coalesced_as_needed\":108},{}]",
+"tippecanoe_decisions": "{\"basezoom\":3,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/pbf/0-0-0-filter-mult.pbf.json

@@ -0,0 +1,19 @@
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "SCALERANK": 0, "NAME": "Los Angeles" }, "geometry": { "type": "Point", "coordinates": [ -118.125000, 33.943360 ] } }
+,
+{ "type": "Feature", "properties": { "SCALERANK": 0, "NAME": "London" }, "geometry": { "type": "Point", "coordinates": [ -0.087891, 51.508742 ] } }
+,
+{ "type": "Feature", "properties": { "SCALERANK": 0, "NAME": "Santiago" }, "geometry": { "type": "Point", "coordinates": [ -70.664062, -33.431441 ] } }
+,
+{ "type": "Feature", "properties": { "SCALERANK": 0, "NAME": "Rome" }, "geometry": { "type": "Point", "coordinates": [ 12.480469, 41.902277 ] } }
+,
+{ "type": "Feature", "properties": { "SCALERANK": 0, "NAME": "Lagos" }, "geometry": { "type": "Point", "coordinates": [ 3.427734, 6.402648 ] } }
+,
+{ "type": "Feature", "properties": { "SCALERANK": 0, "NAME": "Riyadh" }, "geometry": { "type": "Point", "coordinates": [ 46.757812, 24.607069 ] } }
+,
+{ "type": "Feature", "properties": { "SCALERANK": 0, "NAME": "Singapore" }, "geometry": { "type": "Point", "coordinates": [ 103.886719, 1.230374 ] } }
+,
+{ "type": "Feature", "properties": { "SCALERANK": 0, "NAME": "Cape Town" }, "geometry": { "type": "Point", "coordinates": [ 18.457031, -33.943360 ] } }
+] }
+] }

tests/pbf/0-0-0-pop-expr.pbf.json

@@ -0,0 +1,57 @@
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "NAME": "Los Angeles" }, "geometry": { "type": "Point", "coordinates": [ -118.125000, 33.943360 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Mexico City" }, "geometry": { "type": "Point", "coordinates": [ -99.140625, 19.394068 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Washington, D.C." }, "geometry": { "type": "Point", "coordinates": [ -76.992188, 38.891033 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "New York" }, "geometry": { "type": "Point", "coordinates": [ -74.003906, 40.713956 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Bogota" }, "geometry": { "type": "Point", "coordinates": [ -74.091797, 4.565474 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "London" }, "geometry": { "type": "Point", "coordinates": [ -0.087891, 51.508742 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Santiago" }, "geometry": { "type": "Point", "coordinates": [ -70.664062, -33.431441 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Sao Paulo" }, "geometry": { "type": "Point", "coordinates": [ -46.582031, -23.563987 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Rio de Janeiro" }, "geometry": { "type": "Point", "coordinates": [ -43.242188, -22.917923 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Paris" }, "geometry": { "type": "Point", "coordinates": [ 2.373047, 48.864715 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Rome" }, "geometry": { "type": "Point", "coordinates": [ 12.480469, 41.902277 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Istanbul" }, "geometry": { "type": "Point", "coordinates": [ 29.003906, 41.112469 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Moscow" }, "geometry": { "type": "Point", "coordinates": [ 37.617188, 55.727110 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Lagos" }, "geometry": { "type": "Point", "coordinates": [ 3.427734, 6.402648 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Cairo" }, "geometry": { "type": "Point", "coordinates": [ 31.289062, 30.069094 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Riyadh" }, "geometry": { "type": "Point", "coordinates": [ 46.757812, 24.607069 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Kolkata" }, "geometry": { "type": "Point", "coordinates": [ 88.330078, 22.512557 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Mumbai" }, "geometry": { "type": "Point", "coordinates": [ 72.861328, 18.979026 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Singapore" }, "geometry": { "type": "Point", "coordinates": [ 103.886719, 1.230374 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Beijing" }, "geometry": { "type": "Point", "coordinates": [ 116.367188, 39.909736 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Hong Kong" }, "geometry": { "type": "Point", "coordinates": [ 114.169922, 22.268764 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Shanghai" }, "geometry": { "type": "Point", "coordinates": [ 121.464844, 31.203405 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Tokyo" }, "geometry": { "type": "Point", "coordinates": [ 139.746094, 35.675147 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Cape Town" }, "geometry": { "type": "Point", "coordinates": [ 18.457031, -33.943360 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Nairobi" }, "geometry": { "type": "Point", "coordinates": [ 36.826172, -1.318243 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Jakarta" }, "geometry": { "type": "Point", "coordinates": [ 106.875000, -6.227934 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Sydney" }, "geometry": { "type": "Point", "coordinates": [ 151.171875, -33.943360 ] } }
+] }
+] }

tests/pbf/0-0-0-pop-filtered.pbf.json

@@ -0,0 +1,21 @@
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "NAME": "Vancouver" }, "geometry": { "type": "Point", "coordinates": [ -123.134766, 49.267805 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Castries" }, "geometry": { "type": "Point", "coordinates": [ -60.996094, 14.008696 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Nukualofa" }, "geometry": { "type": "MultiPoint", "coordinates": [ [ -175.166016, -21.125498 ], [ 184.833984, -21.125498 ] ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Vienna" }, "geometry": { "type": "Point", "coordinates": [ 16.347656, 48.166085 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Niamey" }, "geometry": { "type": "Point", "coordinates": [ 2.109375, 13.496473 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Hargeysa" }, "geometry": { "type": "Point", "coordinates": [ 44.121094, 9.535749 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Chengdu" }, "geometry": { "type": "Point", "coordinates": [ 104.062500, 30.675715 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Cape Town" }, "geometry": { "type": "Point", "coordinates": [ 18.457031, -33.943360 ] } }
+,
+{ "type": "Feature", "properties": { "NAME": "Suva" }, "geometry": { "type": "MultiPoint", "coordinates": [ [ 178.505859, -18.145852 ], [ -181.582031, -18.145852 ] ] } }
+] }
+] }

tests/pmtiles/hackspots.json

@@ -9,6 +9,7 @@
 "maxzoom": "14",
 "minzoom": "0",
 "name": "tests/pmtiles/hackspots.pmtiles",
+"tippecanoe_decisions": "{\"basezoom\":14,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/raw-tiles/raw-tiles-z67.json

@@ -9,6 +9,7 @@
 "maxzoom": "14",
 "minzoom": "0",
 "name": "tests/raw-tiles/raw-tiles",
+"tippecanoe_decisions": "{\"basezoom\":14,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/raw-tiles/raw-tiles.json

@@ -9,6 +9,7 @@
 "maxzoom": "14",
 "minzoom": "0",
 "name": "tests/raw-tiles/raw-tiles",
+"tippecanoe_decisions": "{\"basezoom\":14,\"droprate\":1,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tests/stable/out/-z3_-B0.json

@@ -9,6 +9,7 @@
 "maxzoom": "3",
 "minzoom": "0",
 "name": "tests/stable/out/-z3_-B0.json.check.mbtiles",
+"tippecanoe_decisions": "{\"basezoom\":0,\"droprate\":2.5,\"retain_points_multiplier\":1}",
 "type": "overlay",
 "version": "2"
 }, "features": [

tile-join.cpp

@@ -155,46 +155,8 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
 			mvt_feature feat = layer.features[f];
 			std::set<std::string> exclude_attributes;
 
-			if (filter != NULL) {
+			if (!evaluate(feat, layer, filter, exclude_attributes, z)) {
-				std::map<std::string, mvt_value> attributes;
+				continue;
-
-				for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) {
-					std::string key = layer.keys[feat.tags[t]];
-					mvt_value &val = layer.values[feat.tags[t + 1]];
-
-					attributes.insert(std::pair<std::string, mvt_value>(key, val));
-				}
-
-				if (feat.has_id) {
-					mvt_value v;
-					v.type = mvt_uint;
-					v.numeric_value.uint_value = feat.id;
-
-					attributes.insert(std::pair<std::string, mvt_value>("$id", v));
-				}
-
-				mvt_value v;
-				v.type = mvt_string;
-
-				if (feat.type == mvt_point) {
-					v.string_value = "Point";
-				} else if (feat.type == mvt_linestring) {
-					v.string_value = "LineString";
-				} else if (feat.type == mvt_polygon) {
-					v.string_value = "Polygon";
-				}
-
-				attributes.insert(std::pair<std::string, mvt_value>("$type", v));
-
-				mvt_value v2;
-				v2.type = mvt_uint;
-				v2.numeric_value.uint_value = z;
-
-				attributes.insert(std::pair<std::string, mvt_value>("$zoom", v2));
-
-				if (!evaluate(attributes, layer.name, filter, exclude_attributes)) {
-					continue;
-				}
 			}
 
 			mvt_feature outfeature;
@@ -782,7 +744,7 @@ struct tileset_reader {
 		}
 
 		if (source.layers.size() != 0) {
-			std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles);
+			std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles, false, NULL);
 			return ret;
 		}
 
@@ -1606,7 +1568,7 @@ int main(int argc, char **argv) {
 		st.maxlon2 = st.maxlon;
 	}
 
-	metadata m = make_metadata(name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.minlat2, st.minlon2, st.maxlat2, st.maxlon2, st.midlat, st.midlon, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options, strategies);
+	metadata m = make_metadata(name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.minlat2, st.minlon2, st.maxlat2, st.maxlon2, st.midlat, st.midlon, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str(), !pg, attribute_descriptions, "tile-join", generator_options, strategies, st.maxzoom, 2.5, 1);
 
 	if (outdb != NULL) {
 		mbtiles_write_metadata(outdb, m, true);

tile.cpp

@@ -1456,7 +1456,11 @@ void remove_attributes(serial_feature &sf, std::set<std::string> const &exclude_
 	}
 }
 
-serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, compressor **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed) {
+struct multiplier_state {
+	std::map<std::string, size_t> count;
+};
+
+serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, compressor **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state) {
 	while (1) {
 		serial_feature sf;
 		std::string s;
@@ -1583,8 +1587,36 @@ serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos
 			}
 		}
 
-		if (sf.tippecanoe_minzoom == -1 && z < sf.feature_minzoom) {
+		if (sf.tippecanoe_minzoom == -1) {
+			bool keep = false;
+
+			std::string layername = (*layer_unmaps)[sf.segment][sf.layer];
+			auto count = multiplier_state->count.find(layername);
+			if (count == multiplier_state->count.end()) {
+				multiplier_state->count.emplace(layername, 0);
+				count = multiplier_state->count.find(layername);
+				keep = true;  // the first feature in each layer in each tile is always kept
+			}
+
 			sf.dropped = true;
+
+			if (z >= sf.feature_minzoom || keep) {
+				count->second = retain_points_multiplier;
+
+				if (retain_points_multiplier > 1) {
+					serial_val val;
+					val.type = mvt_bool;
+					val.s = "true";
+
+					sf.full_keys.push_back("tippecanoe:retain_points_multiplier_first");
+					sf.full_values.push_back(val);
+				}
+			}
+
+			if (count->second > 0) {
+				sf.dropped = false;
+				count->second -= 1;
+			}
 		}
 
 		// Remove nulls, now that the expression evaluation filter has run
@@ -1646,9 +1678,10 @@ struct run_prefilter_args {
 void *run_prefilter(void *v) {
 	run_prefilter_args *rpa = (run_prefilter_args *) v;
 	json_writer state(rpa->prefilter_fp);
+	struct multiplier_state multiplier_state;
 
 	while (1) {
-		serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed);
+		serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed, &multiplier_state);
 		if (sf.t < 0) {
 			break;
 		}
@@ -1841,15 +1874,27 @@ void preserve_attributes(std::map<std::string, attribute_op> const *attribute_ac
 	}
 }
 
-bool find_partial(std::vector<partial> &partials, serial_feature &sf, ssize_t &out, std::vector<std::vector<std::string>> *layer_unmaps, long long maxextent) {
+// This function finds the feature in `partials` onto which the attributes or geometry
+// of a feature that is being dropped (`sf`) will be accumulated or coalesced. It
+// ordinarily returns the most recently-added feature from the same layer as the feature
+// that is being dropped, but if there is an active multiplier, will walk multiple
+// features backward so that the features being dropped will be accumulated round-robin
+// onto the N features that are being kept. The caller increments the `multiplier_seq`
+// mod N with each dropped feature to drive the round-robin decision.
+//
+bool find_partial(std::vector<partial> &partials, serial_feature &sf, ssize_t &out, std::vector<std::vector<std::string>> *layer_unmaps, long long maxextent, ssize_t multiplier_seq) {
 	for (size_t i = partials.size(); i > 0; i--) {
 		if (partials[i - 1].t == sf.t) {
 			std::string &layername1 = (*layer_unmaps)[partials[i - 1].segment][partials[i - 1].layer];
 			std::string &layername2 = (*layer_unmaps)[sf.segment][sf.layer];
 
 			if (layername1 == layername2 && partials[i - 1].extent <= maxextent) {
-				out = i - 1;
+				if (multiplier_seq <= 0) {
-				return true;
+					out = i - 1;
+					return true;
+				}
+
+				multiplier_seq--;
 			}
 		}
 	}
@@ -2069,12 +2114,14 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			prefilter_jp = json_begin_file(prefilter_read_fp);
 		}
 
+		struct multiplier_state multiplier_state;
+		size_t multiplier_seq = retain_points_multiplier - 1;
 		for (size_t seq = 0;; seq++) {
 			serial_feature sf;
 			ssize_t which_partial = -1;
 
 			if (prefilter == NULL) {
-				sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time, compressed_input);
+				sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time, compressed_input, &multiplier_state);
 			} else {
 				sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
 			}
@@ -2098,15 +2145,19 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			}
 
 			if (sf.dropped) {
-				if (find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
+				multiplier_seq = (multiplier_seq + 1) % retain_points_multiplier;
+
+				if (find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX, multiplier_seq)) {
 					preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
 					strategy->dropped_by_rate++;
 					continue;
 				}
+			} else {
+				multiplier_seq = retain_points_multiplier - 1;
 			}
 
 			if (gamma > 0) {
-				if (manage_gap(sf.index, &previndex, scale, gamma, &gap) && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
+				if (manage_gap(sf.index, &previndex, scale, gamma, &gap) && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX, multiplier_seq)) {
 					preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
 					strategy->dropped_by_gamma++;
 					continue;
@@ -2125,7 +2176,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				if (indices.size() < MAX_INDICES) {
 					indices.push_back(sf.index);
 				}
-				if ((sf.index < merge_previndex || sf.index - merge_previndex < mingap) && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
+				if ((sf.index < merge_previndex || sf.index - merge_previndex < mingap) && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX, multiplier_seq)) {
 					partials[which_partial].clustered++;
 
 					if (partials[which_partial].t == VT_POINT &&
@@ -2148,7 +2199,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				if (indices.size() < MAX_INDICES) {
 					indices.push_back(sf.index);
 				}
-				if (sf.index - merge_previndex < mingap && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
+				if (sf.index - merge_previndex < mingap && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX, multiplier_seq)) {
 					preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
 					strategy->dropped_as_needed++;
 					continue;
@@ -2157,7 +2208,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				if (indices.size() < MAX_INDICES) {
 					indices.push_back(sf.index);
 				}
-				if (sf.index - merge_previndex < mingap && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
+				if (sf.index - merge_previndex < mingap && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX, multiplier_seq)) {
 					coalesce_geometry(partials[which_partial], sf);
 					partials[which_partial].coalesced = true;
 					coalesced_area += sf.extent;
@@ -2169,14 +2220,14 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				add_sample_to(extents, sf.extent, extents_increment, seq);
 				// search here is for LLONG_MAX, not minextent, because we are dropping features, not coalescing them,
 				// so we shouldn't expect to find anything small that we can related this feature to.
-				if (minextent != 0 && sf.extent + coalesced_area <= minextent && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
+				if (minextent != 0 && sf.extent + coalesced_area <= minextent && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX, multiplier_seq)) {
 					preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
 					strategy->dropped_as_needed++;
 					continue;
 				}
 			} else if (additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
 				add_sample_to(extents, sf.extent, extents_increment, seq);
-				if (minextent != 0 && sf.extent + coalesced_area <= minextent && find_partial(partials, sf, which_partial, layer_unmaps, minextent)) {
+				if (minextent != 0 && sf.extent + coalesced_area <= minextent && find_partial(partials, sf, which_partial, layer_unmaps, minextent, multiplier_seq)) {
 					coalesce_geometry(partials[which_partial], sf);
 					partials[which_partial].coalesced = true;
 					coalesced_area += sf.extent;
@@ -2199,7 +2250,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			}
 
 			fraction_accum += fraction;
-			if (fraction_accum < 1 && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
+			if (fraction_accum < 1 && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX, multiplier_seq)) {
 				if (additional[A_COALESCE_FRACTION_AS_NEEDED]) {
 					coalesce_geometry(partials[which_partial], sf);
 					partials[which_partial].coalesced = true;

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.40.0"
+#define VERSION "v2.41.0"
 
 #endif