The GeoJSON-producing part of prefiltering

decode.cpp

@@ -56,14 +56,14 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
 			printf("{ \"type\": \"FeatureCollection\"");
 			printf(", \"properties\": { \"layer\": ");
 			fprintq(stdout, layer.name.c_str());
-			printf(", \"version\": %d, \"extent\": %d", layer.version, layer.extent);
+			printf(", \"version\": %d, \"extent\": %lld", layer.version, layer.extent);
 			printf(" }");
 			printf(", \"features\": [\n");
 
 			within = 0;
 		}
 
-		layer_to_geojson(stdout, layer, z, x, y, true);
+		layer_to_geojson(stdout, layer, z, x, y, true, false);
 
 		if (describe) {
 			printf("] }\n");

main.cpp

@@ -947,7 +947,7 @@ void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE
 	}
 }
 
-int read_input(std::vector<source> &sources, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, const char *filter) {
+int read_input(std::vector<source> &sources, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, const char *prefilter, const char *postfilter) {
 	int ret = EXIT_SUCCESS;
 
 	struct reader reader[CPUS];
@@ -1729,7 +1729,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
 	}
 
 	unsigned midx = 0, midy = 0;
-	int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, filter);
+	int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, layermaps, prefilter, postfilter);
 
 	if (maxzoom != written) {
 		fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written);
@@ -1868,7 +1868,8 @@ int main(int argc, char **argv) {
 	const char *tmpdir = "/tmp";
 	const char *attribution = NULL;
 	std::vector<source> sources;
-	const char *filter = NULL;
+	const char *prefilter = NULL;
+	const char *postfilter = NULL;
 
 	std::set<std::string> exclude, include;
 	int exclude_all = 0;
@@ -1904,7 +1905,8 @@ int main(int argc, char **argv) {
 		{"projection", required_argument, 0, 's'},
 		{"simplification", required_argument, 0, 'S'},
 		{"maximum-tile-bytes", required_argument, 0, 'M'},
-		{"filter", required_argument, 0, 'c'},
+		{"prefilter", required_argument, 0, 'C'},
+		{"postfilter", required_argument, 0, 'c'},
 
 		{"exclude-all", no_argument, 0, 'X'},
 		{"force", no_argument, 0, 'f'},
@@ -1960,7 +1962,7 @@ int main(int argc, char **argv) {
 		}
 	}
 
-	while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:c:", long_options, NULL)) != -1) {
+	while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:c:C:", long_options, NULL)) != -1) {
 		switch (i) {
 		case 0:
 			break;
@@ -2144,7 +2146,11 @@ int main(int argc, char **argv) {
 			break;
 
 		case 'c':
-			filter = optarg;
+			postfilter = optarg;
+			break;
+
+		case 'C':
+			prefilter = optarg;
 			break;
 
 		default: {
@@ -2247,7 +2253,7 @@ int main(int argc, char **argv) {
 		sources.push_back(src);
 	}
 
-	ret = read_input(sources, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, filter);
+	ret = read_input(sources, name ? name : outdir, layer, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, prefilter, postfilter);
 
 	mbtiles_close(outdb, argv);
 

mvt.hpp

@@ -65,7 +65,7 @@ struct mvt_layer {
 	std::vector<mvt_feature> features;
 	std::vector<std::string> keys;
 	std::vector<mvt_value> values;
-	int extent;
+	long long extent;
 
 	// Add a key-value pair to a feature, using this layer's constant pool
 	void tag(mvt_feature &feature, std::string key, mvt_value value);

plugin.cpp

@@ -45,7 +45,7 @@ void *run_writer(void *a) {
 		exit(EXIT_FAILURE);
 	}
 
-	layer_to_geojson(fp, *(wa->layer), wa->z, wa->x, wa->y, false);
+	layer_to_geojson(fp, *(wa->layer), wa->z, wa->x, wa->y, false, false);
 
 	if (fclose(fp) != 0) {
 		perror("fclose output to filter");

tile.cpp

@@ -35,6 +35,7 @@
 #include "options.hpp"
 #include "main.hpp"
 #include "plugin.hpp"
+#include "write_json.hpp"
 
 #define CMD_BITS 3
 
@@ -142,7 +143,7 @@ mvt_value retrieve_string(long long off, char *stringpool, int *otype) {
 	return stringified_to_mvt_value(type, s);
 }
 
-void decode_meta(int m, std::vector<long long> &metakeys, std::vector<long long> &metavals, char *stringpool, mvt_layer &layer, mvt_feature &feature) {
+void decode_meta(int m, std::vector<long long> const &metakeys, std::vector<long long> const &metavals, char *stringpool, mvt_layer &layer, mvt_feature &feature) {
 	int i;
 	for (i = 0; i < m; i++) {
 		int otype;
@@ -1167,10 +1168,11 @@ struct write_tile_args {
 	long long minextent_out;
 	double fraction;
 	double fraction_out;
-	const char *filter;
+	const char *prefilter;
+	const char *postfilter;
 };
 
-long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *filter, write_tile_args *arg) {
+long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) {
 	int line_detail;
 	double merge_fraction = 1;
 	double mingap_fraction = 1;
@@ -1422,6 +1424,34 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				continue;
 			}
 
+			if (prefilter != NULL) {
+				mvt_layer tmp_layer;
+				tmp_layer.extent = 1LL << 32;
+				tmp_layer.name = (*layer_unmaps)[segment][layer];
+
+				mvt_feature tmp_feature;
+				tmp_feature.type = t;
+				tmp_feature.geometry = to_feature(geom);
+				tmp_feature.id = id;
+				tmp_feature.has_id = id;
+
+				// Offset from tile coordinates back to world coordinates
+				unsigned sx = 0, sy = 0;
+				if (z != 0) {
+					sx = tx << (32 - z);
+					sy = ty << (32 - z);
+				}
+				for (size_t i = 0; i < tmp_feature.geometry.size(); i++) {
+					tmp_feature.geometry[i].x += sx;
+					tmp_feature.geometry[i].y += sy;
+				}
+
+				decode_meta(m, metakeys, metavals, stringpool + pool_off[segment], tmp_layer, tmp_feature);
+				tmp_layer.features.push_back(tmp_feature);
+
+				layer_to_geojson(stdout, tmp_layer, 0, 0, 0, false, true);
+			}
+
 			if (gamma > 0) {
 				if (manage_gap(index, &previndex, scale, gamma, &gap)) {
 					continue;
@@ -1691,8 +1721,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				layer.features.push_back(feature);
 			}
 
-			if (filter != NULL) {
+			if (postfilter != NULL) {
-				layer = filter_layer(filter, layer, z, tx, ty);
+				layer = filter_layer(postfilter, layer, z, tx, ty);
 			}
 
 			if (layer.features.size() > 0) {
@@ -1921,7 +1951,7 @@ void *run_thread(void *vargs) {
 
 			// fprintf(stderr, "%d/%u/%u\n", z, x, y);
 
-			long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->filter, arg);
+			long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg);
 
 			if (len < 0) {
 				int *err = &arg->err;
@@ -1986,7 +2016,7 @@ void *run_thread(void *vargs) {
 	return NULL;
 }
 
-int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *filter) {
+int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter) {
 	// Table to map segment and layer number back to layer name
 	std::vector<std::vector<std::string>> layer_unmaps;
 	for (size_t seg = 0; seg < layermaps.size(); seg++) {
@@ -2153,7 +2183,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
 				args[thread].initial_y = initial_y;
 				args[thread].layermaps = &layermaps;
 				args[thread].layer_unmaps = &layer_unmaps;
-				args[thread].filter = filter;
+				args[thread].prefilter = prefilter;
+				args[thread].postfilter = postfilter;
 
 				args[thread].tasks = dispatches[thread].tasks;
 				args[thread].running = &running;

tile.hpp

@@ -1,3 +1,3 @@
-int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *filter);
+int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter);
 
 int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);

write_json.cpp

@@ -24,7 +24,7 @@ struct lonlat {
 	}
 };
 
-void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsigned y, bool comma) {
+void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name) {
 	for (size_t f = 0; f < layer.features.size(); f++) {
 		mvt_feature &feat = layer.features[f];
 
@@ -38,6 +38,12 @@ void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsign
 			fprintf(fp, ", \"id\": %llu", feat.id);
 		}
 
+		if (name) {
+			fprintf(fp, ", \"tippecanoe\": { \"layer\": ");
+			fprintq(fp, layer.name.c_str());
+			fprintf(fp, " }");
+		}
+
 		fprintf(fp, ", \"properties\": { ");
 
 		for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) {

write_json.hpp

@@ -1,2 +1,2 @@
-void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsigned y, bool comma);
+void layer_to_geojson(FILE *fp, mvt_layer &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name);
 void fprintq(FILE *f, const char *s);