Accept tilestats limiting options in tile-join, not just tippecanoe (#8)

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.4.1
+
+* Accept tilestats limiting options in tile-join, not just tippecanoe
+
 ## 2.4.0
 
 * Change maxzoom guessing to take into account the standard deviation of the distances between features, so data with tight clusters will choose a higher maxzoom

Makefile

@@ -197,18 +197,27 @@ join-test: tile-join
 	./tile-join --quiet --force -o tests/join-population/joined.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join --quiet --force -o tests/join-population/joined-null.mbtiles --empty-csv-columns-are-null -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join --quiet --force --no-tile-stats -o tests/join-population/joined-no-tile-stats.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
+	./tile-join --quiet --force --tile-stats-attributes-limit=1 -o tests/join-population/joined-tile-stats-attributes-limit.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
+	./tile-join --quiet --force --tile-stats-sample-values-limit=1 -o tests/join-population/joined-tile-stats-sample-values-limit.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
+	./tile-join --quiet --force --tile-stats-values-limit=1 -o tests/join-population/joined-tile-stats-values-limit.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join -q -f -i -o tests/join-population/joined-i.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join -q -f -o tests/join-population/merged.mbtiles tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
 	./tile-join -q -f -c tests/join-population/windows.csv -o tests/join-population/windows.mbtiles tests/join-population/macarthur.mbtiles
 	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined.mbtiles > tests/join-population/joined.mbtiles.json.check
 	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-null.mbtiles > tests/join-population/joined-null.mbtiles.json.check
 	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-no-tile-stats.mbtiles > tests/join-population/joined-no-tile-stats.mbtiles.json.check
+	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-tile-stats-attributes-limit.mbtiles > tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json.check
+	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-tile-stats-values-limit.mbtiles > tests/join-population/joined-tile-stats-values-limit.mbtiles.json.check
+	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-tile-stats-sample-values-limit.mbtiles > tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json.check
 	./tippecanoe-decode -x generator tests/join-population/joined-i.mbtiles > tests/join-population/joined-i.mbtiles.json.check
 	./tippecanoe-decode -x generator tests/join-population/merged.mbtiles > tests/join-population/merged.mbtiles.json.check
 	./tippecanoe-decode -x generator tests/join-population/windows.mbtiles > tests/join-population/windows.mbtiles.json.check
 	cmp tests/join-population/joined.mbtiles.json.check tests/join-population/joined.mbtiles.json
 	cmp tests/join-population/joined-null.mbtiles.json.check tests/join-population/joined-null.mbtiles.json
 	cmp tests/join-population/joined-no-tile-stats.mbtiles.json.check tests/join-population/joined-no-tile-stats.mbtiles.json
+	cmp tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json.check tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json
+	cmp tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json.check tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json
+	cmp tests/join-population/joined-tile-stats-values-limit.mbtiles.json.check tests/join-population/joined-tile-stats-values-limit.mbtiles.json
 	cmp tests/join-population/joined-i.mbtiles.json.check tests/join-population/joined-i.mbtiles.json
 	cmp tests/join-population/merged.mbtiles.json.check tests/join-population/merged.mbtiles.json
 	cmp tests/join-population/windows.mbtiles.json.check tests/join-population/windows.mbtiles.json

README.md

@@ -779,6 +779,9 @@ The options are:
  * `-pk` or `--no-tile-size-limit`: Don't skip tiles larger than 500K.
  * `-pC` or `--no-tile-compression`: Don't compress the PBF vector tile data.
  * `-pg` or `--no-tile-stats`: Don't generate the `tilestats` row in the tileset metadata. Uploads without [tilestats](https://github.com/mapbox/mapbox-geostats) will take longer to process.
+ * `--tile-stats-attributes-limit=`*count*: Include `tilestats` information about at most *count* attributes instead of the default 1000.
+ * `--tile-stats-sample-values-limit=`*count*: Calculate `tilestats` attribute statistics based on *count* values instead of the default 1000.
+ * `--tile-stats-values-limit=`*count*: Report *count* unique attribute values in `tilestats` instead of the default 100.
 
 Because tile-join just copies the geometries to the new .mbtiles without processing them
 (except to rescale the extents if necessary),

man/tippecanoe.1

@@ -979,6 +979,12 @@ The options are:
 \fB\fC\-pC\fR or \fB\fC\-\-no\-tile\-compression\fR: Don't compress the PBF vector tile data.
 .IP \(bu 2
 \fB\fC\-pg\fR or \fB\fC\-\-no\-tile\-stats\fR: Don't generate the \fB\fCtilestats\fR row in the tileset metadata. Uploads without tilestats \[la]https://github.com/mapbox/mapbox-geostats\[ra] will take longer to process.
+.IP \(bu 2
+\fB\fC\-\-tile\-stats\-attributes\-limit=\fR\fIcount\fP: Include \fB\fCtilestats\fR information about at most \fIcount\fP attributes instead of the default 1000.
+.IP \(bu 2
+\fB\fC\-\-tile\-stats\-sample\-values\-limit=\fR\fIcount\fP: Calculate \fB\fCtilestats\fR attribute statistics based on \fIcount\fP values instead of the default 1000.
+.IP \(bu 2
+\fB\fC\-\-tile\-stats\-values\-limit=\fR\fIcount\fP: Report \fIcount\fP unique attribute values in \fB\fCtilestats\fR instead of the default 100.
 .RE
 .PP
 Because tile\-join just copies the geometries to the new .mbtiles without processing them

mbtiles.cpp

@@ -500,6 +500,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 				state.nospace = true;
 
 				bool first = true;
+				size_t attribute_count = 0;
 				for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) {
 					if (first) {
 						first = false;
@@ -526,6 +527,11 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 					} else {
 						state.json_write_string(f->second);
 					}
+
+					attribute_count++;
+					if (attribute_count >= max_tilestats_attributes) {
+						break;
+					}
 				}
 
 				state.nospace = true;

tile-join.cpp

@@ -909,6 +909,9 @@ int main(int argc, char **argv) {
 		{"no-tile-compression", no_argument, &pC, 1},
 		{"empty-csv-columns-are-null", no_argument, &pe, 1},
 		{"no-tile-stats", no_argument, &pg, 1},
+		{"tile-stats-attributes-limit", required_argument, 0, '~'},
+		{"tile-stats-sample-values-limit", required_argument, 0, '~'},
+		{"tile-stats-values-limit", required_argument, 0, '~'},
 
 		{0, 0, 0, 0},
 	};
@@ -930,7 +933,8 @@ int main(int argc, char **argv) {
 
 	std::string commandline = format_commandline(argc, argv);
 
-	while ((i = getopt_long(argc, argv, getopt_str.c_str(), long_options, NULL)) != -1) {
+	int option_index = 0;
+	while ((i = getopt_long(argc, argv, getopt_str.c_str(), long_options, &option_index)) != -1) {
 		switch (i) {
 		case 0:
 			break;
@@ -1036,6 +1040,21 @@ int main(int argc, char **argv) {
 			quiet = true;
 			break;
 
+		case '~': {
+			const char *opt = long_options[option_index].name;
+			if (strcmp(opt, "tile-stats-attributes-limit") == 0) {
+				max_tilestats_attributes = atoi(optarg);
+			} else if (strcmp(opt, "tile-stats-sample-values-limit") == 0) {
+				max_tilestats_sample_values = atoi(optarg);
+			} else if (strcmp(opt, "tile-stats-values-limit") == 0) {
+				max_tilestats_values = atoi(optarg);
+			} else {
+				fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt);
+				exit(EXIT_FAILURE);
+			}
+			break;
+		}
+
 		default:
 			usage(argv);
 		}

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.4.0"
+#define VERSION "v2.4.1"
 
 #endif