Add --drop-by-attribute-as-needed option (#384)

README.md

@@ -482,6 +482,7 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
  * `-as` or `--drop-densest-as-needed`: If a tile is too large, try to reduce it to under 500K by increasing the minimum spacing between features. The discovered spacing applies to the entire zoom level.
  * `-ad` or `--drop-fraction-as-needed`: Dynamically drop some fraction of features from each zoom level to keep large tiles under the 500K size limit. (This is like `-pd` but applies to the entire zoom level, not to each tile.)
  * `-an` or `--drop-smallest-as-needed`: Dynamically drop the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level to keep large tiles under the 500K size limit.
+ * `--drop-by-attribute-as-needed=`_attribute_: Dynamically drop features with the lowest values of the specified numeric _attribute_ from each zoom level to keep large tiles under the 500K size limit. Use `--drop-by-attribute-order=desc` to instead drop features with the highest values.
  * `-aN` or `--coalesce-smallest-as-needed`: Dynamically combine the smallest features (physically smallest: the shortest lines or the smallest polygons or the densest points) from each zoom level into other nearby features to keep large tiles under the 500K size limit. This option will probably not help very much with LineStrings. It is mostly intended for polygons, to maintain the full original area covered by polygons while still reducing the feature count somehow. The attributes of the small polygons are *not* preserved into the combined features (except through `--accumulate-attribute`), only their geometry. Furthermore, the polygons to which nested polygons are coalesced may not necessarily be the immediately enclosing features.
  * `-aD` or `--coalesce-densest-as-needed`: Dynamically combine the densest features from each zoom level into other nearby features to keep large tiles under the 500K size limit. (Again, mostly useful for polygons.)
  * `-aS` or `--coalesce-fraction-as-needed`: Dynamically combine a fraction of features from each zoom level into other nearby features to keep large tiles under the 500K size limit. (Again, mostly useful for polygons.)

main.cpp

@@ -98,6 +98,8 @@ long long extend_zooms_max = 0;
 int retain_points_multiplier = 1;
 std::vector<std::string> unidecode_data;
 size_t maximum_string_attribute_length = 0;
+std::string drop_by_attribute_as_needed_attribute;
+bool drop_by_attribute_descending = false;
 
 std::vector<order_field> order_by;
 bool order_reverse;
@@ -2754,7 +2756,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 	std::atomic<unsigned> midx(0);
 	std::atomic<unsigned> midy(0);
 	std::vector<strategy> strategies;
-	int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz, shared_nodes_map, nodepos, shared_nodes_bloom, basezoom, droprate, unidecode_data);
+	int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz, shared_nodes_map, nodepos, shared_nodes_bloom, basezoom, droprate, unidecode_data, &drop_by_attribute_as_needed_attribute, drop_by_attribute_descending);
 
 	if (maxzoom != written) {
 		if (written > minzoom) {
@@ -3090,6 +3092,8 @@ int main(int argc, char **argv) {
 		{"drop-densest-as-needed", no_argument, &additional[A_DROP_DENSEST_AS_NEEDED], 1},
 		{"drop-fraction-as-needed", no_argument, &additional[A_DROP_FRACTION_AS_NEEDED], 1},
 		{"drop-smallest-as-needed", no_argument, &additional[A_DROP_SMALLEST_AS_NEEDED], 1},
+		{"drop-by-attribute-as-needed", required_argument, 0, '~'},
+		{"drop-by-attribute-order", required_argument, 0, '~'},
 		{"coalesce-densest-as-needed", no_argument, &additional[A_COALESCE_DENSEST_AS_NEEDED], 1},
 		{"coalesce-fraction-as-needed", no_argument, &additional[A_COALESCE_FRACTION_AS_NEEDED], 1},
 		{"coalesce-smallest-as-needed", no_argument, &additional[A_COALESCE_SMALLEST_AS_NEEDED], 1},
@@ -3278,6 +3282,22 @@ int main(int argc, char **argv) {
 			} else if (strcmp(opt, "order-largest-first") == 0) {
 				order_by.push_back(order_field(ORDER_BY_SIZE, true));
 				order_by_size = true;
+			} else if (strcmp(opt, "drop-by-attribute-as-needed") == 0) {
+				if (optarg[0] == '\0') {
+					fprintf(stderr, "%s: --drop-by-attribute-as-needed requires a non-empty attribute name\n", argv[0]);
+					exit(EXIT_ARGS);
+				}
+				drop_by_attribute_as_needed_attribute = optarg;
+				additional[A_DROP_BY_ATTRIBUTE_AS_NEEDED] = 1;
+			} else if (strcmp(opt, "drop-by-attribute-order") == 0) {
+				if (strcmp(optarg, "asc") == 0) {
+					drop_by_attribute_descending = false;
+				} else if (strcmp(optarg, "desc") == 0) {
+					drop_by_attribute_descending = true;
+				} else {
+					fprintf(stderr, "%s: %s: order must be one of: asc, desc\n", argv[0], optarg);
+					exit(EXIT_ARGS);
+				}
 			} else if (strcmp(opt, "simplification-at-maximum-zoom") == 0) {
 				maxzoom_simplification = atof_require(optarg, "Mazoom simplification");
 				if (maxzoom_simplification <= 0) {
@@ -3799,6 +3819,11 @@ int main(int argc, char **argv) {
 		check_dir(out_dir, argv, force, forcetable);
 	}
 
+	if (additional[A_DROP_BY_ATTRIBUTE_AS_NEEDED] && drop_by_attribute_as_needed_attribute.empty()) {
+		fprintf(stderr, "%s: --drop-by-attribute-as-needed requires an attribute name\n", argv[0]);
+		exit(EXIT_ARGS);
+	}
+
 	int ret = EXIT_SUCCESS;
 
 	for (i = optind; i < argc; i++) {

options.hpp

@@ -29,6 +29,7 @@
 #define A_VISVALINGAM ((int) 'v')
 #define A_DETECT_WRAPAROUND ((int) 'w')
 #define A_KEEP_POINT_CLUSTER_POSITION ((int) 'a')
+#define A_DROP_BY_ATTRIBUTE_AS_NEEDED ((int) 'A')
 
 #define P_TILE_COMPRESSION ((int) 'C')
 #define P_DUPLICATION ((int) 'D')

tile.cpp

@@ -825,6 +825,44 @@ static unsigned long long choose_mindrop_sequence(std::vector<unsigned long long
 	return drop_sequences[ix];
 }
 
+static double choose_minattribute(std::vector<double> &attribute_values, double f, double existing_attribute, bool descending) {
+	if (attribute_values.size() == 0) {
+		return existing_attribute;
+	}
+
+	std::stable_sort(attribute_values.begin(), attribute_values.end());
+
+	if (descending) {
+		// For descending: threshold moves down, drop features >= threshold
+		// Use ceil so ix points at the first value to drop, not the last to keep
+		size_t ix = (size_t)ceil((double)(attribute_values.size() - 1) * f);
+		if (ix >= attribute_values.size()) {
+			ix = attribute_values.size() - 1;
+		}
+		while (ix > 0 && attribute_values[ix] >= existing_attribute) {
+			ix--;
+		}
+
+		if (attribute_values[ix] >= existing_attribute) {
+			return existing_attribute;
+		}
+
+		return attribute_values[ix];
+	} else {
+		// For ascending: threshold moves up, drop features <= threshold
+		size_t ix = (attribute_values.size() - 1) * (1 - f);
+		while (ix + 1 < attribute_values.size() && attribute_values[ix] <= existing_attribute) {
+			ix++;
+		}
+
+		if (attribute_values[ix] <= existing_attribute) {
+			return existing_attribute;
+		}
+
+		return attribute_values[ix];
+	}
+}
+
 static unsigned long long calculate_drop_sequence(serial_feature const &sf) {
 	unsigned long long zoom = std::min(std::max((unsigned long long) sf.feature_minzoom, 0ULL), 31ULL);
 	unsigned long long out = zoom << (64 - 5);	      // top bits are the zoom level: top-priority features are those that appear in the low zooms
@@ -890,6 +928,10 @@ struct write_tile_args {
 	long long minextent_out = 0;
 	unsigned long long mindrop_sequence = 0;
 	unsigned long long mindrop_sequence_out = 0;
+	double minattribute = 0;
+	double minattribute_out = 0;
+	std::string const *drop_by_attribute_as_needed_attribute = NULL;
+	bool drop_by_attribute_descending = false;
 	size_t tile_size_out = 0;
 	size_t feature_count_out = 0;
 	const char *prefilter = NULL;
@@ -1598,11 +1640,12 @@ void skip_tile(decompressor *geoms, std::atomic<long long> *geompos_in, bool com
 	}
 }
 
-long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector<std::string> const &unidecode_data, long long estimated_complexity, std::set<zxy> &skip_children_out) {
+long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, double minattribute, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector<std::string> const &unidecode_data, long long estimated_complexity, std::set<zxy> &skip_children_out) {
 	double merge_fraction = 1;
 	double mingap_fraction = 1;
 	double minextent_fraction = 1;
 	double mindrop_sequence_fraction = 1;
+	double minattribute_fraction = 1;
 
 	static std::atomic<double> oprogress(0);
 	long long og = *geompos_in;
@@ -1676,6 +1719,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 		size_t extents_increment = 1;
 		std::vector<unsigned long long> drop_sequences;
 		size_t drop_sequences_increment = 1;
+		std::vector<double> attribute_values;
+		size_t attribute_values_increment = 1;
 
 		double coalesced_area = 0;
 		drawvec shared_nodes;
@@ -2039,6 +2084,40 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						can_stop_early = false;
 						continue;
 					}
+				} else if (additional[A_DROP_BY_ATTRIBUTE_AS_NEEDED]) {
+					mvt_value attr_val = find_attribute_value(&sf, *arg->drop_by_attribute_as_needed_attribute);
+					double attr_numeric = 0;
+					bool attr_valid = false;
+
+					if (attr_val.type == mvt_double) {
+						attr_numeric = attr_val.numeric_value.double_value;
+						attr_valid = std::isfinite(attr_numeric);
+					} else if (attr_val.type == mvt_float) {
+						attr_numeric = attr_val.numeric_value.float_value;
+						attr_valid = std::isfinite(attr_numeric);
+					} else if (attr_val.type == mvt_int) {
+						attr_numeric = attr_val.numeric_value.int_value;
+						attr_valid = true;
+					} else if (attr_val.type == mvt_uint) {
+						attr_numeric = attr_val.numeric_value.uint_value;
+						attr_valid = true;
+					} else if (attr_val.type == mvt_sint) {
+						attr_numeric = attr_val.numeric_value.sint_value;
+						attr_valid = true;
+					}
+
+					if (attr_valid) {
+						add_sample_to(attribute_values, attr_numeric, attribute_values_increment, seq);
+						bool should_drop = arg->drop_by_attribute_descending
+							? (minattribute != HUGE_VAL && attr_numeric >= minattribute)
+							: (minattribute != -HUGE_VAL && attr_numeric <= minattribute);
+						if (should_drop) {
+							can_stop_early = false;
+							if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum, key_pool)) {
+								continue;
+							}
+						}
+					}
 				}
 			}
 
@@ -2692,6 +2771,45 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						fprintf(stderr, "Can't increase feature area threshold further\n");
 						exit(EXIT_INCOMPLETE);
 					}
+				} else if (additional[A_DROP_BY_ATTRIBUTE_AS_NEEDED]) {
+					minattribute_fraction = minattribute_fraction *
+									adjusted_max_tile_features / adjusted_feature_count * 0.75;
+					if (minattribute_fraction > 0.80) {
+						if (!quiet) {
+							fprintf(stderr,
+								"Need to drop features, but calculated that we should keep %.1f%% of features\n",
+								minattribute_fraction * 100.0);
+						}
+						minattribute_fraction = 0.80;
+					}
+
+					bool desc = arg->drop_by_attribute_descending;
+					if (attribute_values.empty() && !quiet) {
+						fprintf(stderr, "Warning: no features had a numeric value for attribute '%s'\n",
+							arg->drop_by_attribute_as_needed_attribute->c_str());
+					}
+					double m = choose_minattribute(attribute_values, minattribute_fraction, minattribute, desc);
+					bool better = desc ? m < minattribute : m > minattribute;
+					if (better) {
+						minattribute = m;
+						bool propagate = desc ? minattribute < arg->minattribute_out : minattribute > arg->minattribute_out;
+						if (propagate) {
+							arg->minattribute_out = minattribute;
+							arg->still_dropping = true;
+						}
+						if (!quiet) {
+							fprintf(stderr,
+								"Trying to keep features with '%s' %s %.6f to make it fit\n",
+								arg->drop_by_attribute_as_needed_attribute->c_str(),
+								desc ? "<" : ">",
+								minattribute);
+						}
+						line_detail++;
+						continue;
+					} else {
+						fprintf(stderr, "Can't increase attribute threshold further\n");
+						exit(EXIT_INCOMPLETE);
+					}
 				} else if (feature_count > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
 					// The 95% is a guess to avoid too many retries
 					// and probably actually varies based on how much duplicated metadata there is
@@ -2829,6 +2947,37 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						fprintf(stderr, "Can't increase feature area threshold further\n");
 						exit(EXIT_INCOMPLETE);
 					}
+				} else if (additional[A_DROP_BY_ATTRIBUTE_AS_NEEDED]) {
+					minattribute_fraction = minattribute_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75;
+					if (minattribute_fraction > 0.80) {
+						if (!quiet) {
+							fprintf(stderr, "Need to drop features, but calculated that we should keep %.1f%% of features\n", minattribute_fraction * 100.0);
+						}
+						minattribute_fraction = 0.80;
+					}
+					bool desc2 = arg->drop_by_attribute_descending;
+					if (attribute_values.empty() && !quiet) {
+						fprintf(stderr, "Warning: no features had a numeric value for attribute '%s'\n",
+							arg->drop_by_attribute_as_needed_attribute->c_str());
+					}
+					double m = choose_minattribute(attribute_values, minattribute_fraction, minattribute, desc2);
+					bool better2 = desc2 ? m < minattribute : m > minattribute;
+					if (better2) {
+						minattribute = m;
+						bool propagate2 = desc2 ? minattribute < arg->minattribute_out : minattribute > arg->minattribute_out;
+						if (propagate2) {
+							arg->minattribute_out = minattribute;
+							arg->still_dropping = true;
+						}
+						if (!quiet) {
+							fprintf(stderr, "Going to try keeping features with attribute '%s' %s %0.6f to make it fit\n", arg->drop_by_attribute_as_needed_attribute->c_str(), desc2 ? "<" : ">", minattribute);
+						}
+						line_detail++;
+						continue;
+					} else {
+						fprintf(stderr, "Can't increase attribute threshold further\n");
+						exit(EXIT_INCOMPLETE);
+					}
 				} else if (feature_count > layers.size() && (additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED] || prevent[P_DYNAMIC_DROP])) {
 					mindrop_sequence_fraction = mindrop_sequence_fraction * adjusted_max_tile_size / adjusted_tile_size * 0.75;
 					if (mindrop_sequence_fraction > 0.80) {
@@ -2997,7 +3146,7 @@ exit(EXIT_IMPOSSIBLE);
 				len = 1;
 			} else {
 				arg->wrote_zoom = z;
-				len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out);
+				len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->minattribute, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out);
 			}
 
 			if (pthread_mutex_lock(&var_lock) != 0) {
@@ -3063,7 +3212,7 @@ exit(EXIT_IMPOSSIBLE);
 	return err_or_null;
 }
 
-int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, json_object *filter, std::vector<strategy> &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector<std::string> const &unidecode_data) {
+int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, json_object *filter, std::vector<strategy> &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector<std::string> const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending) {
 	last_progress = 0;
 
 	// The existing layermaps are one table per input thread.
@@ -3178,6 +3327,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 		unsigned long long zoom_mingap = 0;
 		long long zoom_minextent = 0;
 		unsigned long long zoom_mindrop_sequence = 0;
+		double zoom_minattribute = drop_by_attribute_descending ? HUGE_VAL : -HUGE_VAL;
 		size_t zoom_tile_size = 0;
 		size_t zoom_feature_count = 0;
 		std::set<zxy> skip_children_out;
@@ -3214,6 +3364,10 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 				args[thread].minextent_out = zoom_minextent;
 				args[thread].mindrop_sequence = zoom_mindrop_sequence;
 				args[thread].mindrop_sequence_out = zoom_mindrop_sequence;
+				args[thread].minattribute = zoom_minattribute;
+				args[thread].minattribute_out = zoom_minattribute;
+				args[thread].drop_by_attribute_as_needed_attribute = drop_by_attribute_as_needed_attribute;
+				args[thread].drop_by_attribute_descending = drop_by_attribute_descending;
 				args[thread].tile_size_out = 0;
 				args[thread].feature_count_out = 0;
 				args[thread].child_shards = TEMP_FILES / threads;
@@ -3300,6 +3454,13 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 					zoom_mindrop_sequence = args[thread].mindrop_sequence_out;
 					again = true;
 				}
+				bool attr_propagate = drop_by_attribute_descending
+						? args[thread].minattribute_out < zoom_minattribute
+						: args[thread].minattribute_out > zoom_minattribute;
+				if (attr_propagate) {
+					zoom_minattribute = args[thread].minattribute_out;
+					again = true;
+				}
 				if (args[thread].tile_size_out > zoom_tile_size) {
 					zoom_tile_size = args[thread].tile_size_out;
 				}

tile.hpp

@@ -62,7 +62,7 @@ struct strategy {
 
 // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
 
-int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector<std::string> const &unidecode_data);
+int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector<std::string> const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending);
 
 int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);