Merge branch 'master' into multithread

Conflicts:
	tile.cc

Makefile

@@ -1,7 +1,9 @@
 PREFIX ?= /usr/local
 MANDIR ?= /usr/share/man/man1/
 
-all: tippecanoe enumerate decode man/tippecanoe.1
+all: tippecanoe enumerate decode tile-join
+
+docs: man/tippecanoe.1
 
 install: tippecanoe
 	mkdir -p $(PREFIX)/bin
@@ -31,6 +33,9 @@ enumerate: enumerate.o
 decode: decode.o vector_tile.pb.o projection.o
 	g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3
 
+tile-join: tile-join.o vector_tile.pb.o projection.o pool.o mbtiles.o
+	g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3
+
 libjsonpull.a: jsonpull.o
 	ar rc $@ $^
 	ranlib $@

README.md

@@ -61,12 +61,13 @@ Options
 
  * -o _file_.mbtiles: Name the output file.
  * -f: Delete the mbtiles file if it already exists instead of giving an error
+ * -t _directory_: Put the temporary files in _directory_.
 
 ### Zoom levels and resolution
 
  * -z _zoom_: Base (maxzoom) zoom level (default 14)
  * -Z _zoom_: Lowest (minzoom) zoom level (default 0)
- * -d _detail_: Detail at base zoom level (default 26-basezoom, ~0.5m, for tile resolution of 4096 if -z14)
+ * -d _detail_: Detail at base zoom level (default 12 at -z14 or higher, or 13 at -z13 or lower. Detail beyond 13 has rendering problems with Mapbox GL.)
  * -D _detail_: Detail at lower zoom levels (default 10, for tile resolution of 1024)
  * -m _detail_: Minimum detail that it will try if tiles are too big at regular detail (default 7)
  * -b _pixels_: Buffer size where features are duplicated from adjacent tiles. Units are "screen pixels"--1/256th of the tile width or height. (default 5)
@@ -82,15 +83,18 @@ Options
  * -r _rate_: Rate at which dots are dropped at lower zoom levels (default 2.5)
  * -g _gamma_: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number.
 
+### Doing more
+
+ * -ac: Coalesce adjacent line and polygon features that have the same properties
+ * -ar: Try reversing the directions of lines to make them coalesce and compress better
+ * -ao: Reorder features to put ones with the same properties in sequence, to try to get them to coalesce
+ * -al: Let "dot" dropping at lower zooms apply to lines too
+
 ### Doing less
 
  * -ps: Don't simplify lines
- * -pr: Don't reverse the direction of lines to make them coalesce better
- * -pc: Don't coalesce features with the same properties
  * -pf: Don't limit tiles to 200,000 features
  * -pk: Don't limit tiles to 500K bytes
- * -po: Don't reorder features to put the same properties in sequence
- * -pl: Let "dot" simplification apply to lines too
  * -pd: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries.
  * -q: Work quietly instead of reporting progress
 
@@ -143,7 +147,7 @@ I don't know why 2.5 is the appropriate number, but the densities of many differ
 data sets fall off at about this same rate. You can use -r to specify a different rate.
 
 You can use the gamma option to thin out especially dense clusters of points.
-For any area that where dots are closer than one pixel together (at whatever zoom level),
+For any area where dots are closer than one pixel together (at whatever zoom level),
 a gamma of 3, for example, will reduce these clusters to the cube root of their original density.
 
 For line features, it drops any features that are too small to draw at all.
@@ -165,10 +169,20 @@ lower resolutions before failing if it still doesn't fit.
 Development
 -----------
 
-Requires protoc (`brew install protobuf` or
+Requires protoc and sqlite3. Rebuilding the manpage
-`apt-get install libprotobuf-dev` and `protobuf-compiler`),
+uses md2man (`gem install md2man`).
-`md2man` (`gem install md2man`), and sqlite3 (`apt-get install libsqlite3-dev`).
+
-To build:
+MacOS:
+
+    brew install protobuf
+
+Linux:
+
+    sudo apt-get install libprotobuf-dev
+    sudo apt-get install protobuf-compiler
+    sudo apt-get install libsqlite3-dev
+
+Then build:
 
     make
 
@@ -184,4 +198,4 @@ Check out [some examples of maps made with tippecanoe](MADE_WITH.md)
 Name
 ----
 
-The name is [a joking reference](http://en.wikipedia.org/wiki/Tippecanoe_and_Tyler_Too) to making tiles.
+The name is [a joking reference](http://en.wikipedia.org/wiki/Tippecanoe_and_Tyler_Too) to a "tiler" for making map tiles.

geojson.c

@@ -1,3 +1,7 @@
+#ifdef MTRACE
+#include <mcheck.h>
+#endif
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <math.h>
@@ -405,7 +409,160 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, char *s, c
 	return off;
 }
 
-int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent) {
+int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, json_pull *jp, long long *seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int maxzoom, int n, double droprate, unsigned *file_bbox) {
+	json_object *geometry_type = json_hash_get(geometry, "type");
+	if (geometry_type == NULL) {
+		static int warned = 0;
+		if (!warned) {
+			fprintf(stderr, "%s:%d: null geometry (additional not reported)\n", reading, jp->line);
+			warned = 1;
+		}
+
+		return 0;
+	}
+
+	if (geometry_type->type != JSON_STRING) {
+		fprintf(stderr, "%s:%d: geometry without type\n", reading, jp->line);
+		return 0;
+	}
+
+	json_object *coordinates = json_hash_get(geometry, "coordinates");
+	if (coordinates == NULL || coordinates->type != JSON_ARRAY) {
+		fprintf(stderr, "%s:%d: feature without coordinates array\n", reading, jp->line);
+		return 0;
+	}
+
+	int t;
+	for (t = 0; t < GEOM_TYPES; t++) {
+		if (strcmp(geometry_type->string, geometry_names[t]) == 0) {
+			break;
+		}
+	}
+	if (t >= GEOM_TYPES) {
+		fprintf(stderr, "%s:%d: Can't handle geometry type %s\n", reading, jp->line, geometry_type->string);
+		return 0;
+	}
+
+	unsigned bbox[] = {UINT_MAX, UINT_MAX, 0, 0};
+
+	int nprop = 0;
+	if (properties->type == JSON_HASH) {
+		nprop = properties->length;
+	}
+
+	long long metastart = *metapos;
+	char *metakey[nprop];
+	char *metaval[nprop];
+	int metatype[nprop];
+	int m = 0;
+
+	int i;
+	for (i = 0; i < nprop; i++) {
+		if (properties->keys[i]->type == JSON_STRING) {
+			if (exclude_all) {
+				if (!is_pooled(include, properties->keys[i]->string, VT_STRING)) {
+					continue;
+				}
+			} else if (is_pooled(exclude, properties->keys[i]->string, VT_STRING)) {
+				continue;
+			}
+
+			metakey[m] = properties->keys[i]->string;
+
+			if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) {
+				metatype[m] = VT_STRING;
+				metaval[m] = properties->values[i]->string;
+				m++;
+			} else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) {
+				metatype[m] = VT_NUMBER;
+				metaval[m] = properties->values[i]->string;
+				m++;
+			} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) {
+				metatype[m] = VT_BOOLEAN;
+				metaval[m] = properties->values[i]->type == JSON_TRUE ? "true" : "false";
+				m++;
+			} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) {
+				;
+			} else {
+				fprintf(stderr, "%s:%d: Unsupported property type for %s\n", reading, jp->line, properties->keys[i]->string);
+				continue;
+			}
+		}
+	}
+
+	serialize_int(metafile, m, metapos, fname);
+	for (i = 0; i < m; i++) {
+		serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname);
+		serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname);
+	}
+
+	long long geomstart = *geompos;
+
+	serialize_byte(geomfile, mb_geometry[t], geompos, fname);
+	serialize_long_long(geomfile, n, geompos, fname);
+	serialize_long_long(geomfile, metastart, geompos, fname);
+	long long wx = initial_x, wy = initial_y;
+	parse_geometry(t, coordinates, bbox, geompos, geomfile, VT_MOVETO, fname, jp, &wx, &wy, &initialized);
+	serialize_byte(geomfile, VT_END, geompos, fname);
+
+	/*
+	 * Note that minzoom for lines is the dimension
+	 * of the geometry in world coordinates, but
+	 * for points is the lowest zoom level (in tiles,
+	 * not in pixels) at which it should be drawn.
+	 *
+	 * So a line that is too small for, say, z8
+	 * will have minzoom of 18 (if tile detail is 10),
+	 * not 8.
+	 */
+	int minzoom = 0;
+	if (mb_geometry[t] == VT_LINE) {
+		for (minzoom = 0; minzoom < 31; minzoom++) {
+			unsigned mask = 1 << (32 - (minzoom + 1));
+
+			if (((bbox[0] & mask) != (bbox[2] & mask)) || ((bbox[1] & mask) != (bbox[3] & mask))) {
+				break;
+			}
+		}
+	} else if (mb_geometry[t] == VT_POINT) {
+		double r = ((double) rand()) / RAND_MAX;
+		if (r == 0) {
+			r = .00000001;
+		}
+		minzoom = maxzoom - floor(log(r) / -log(droprate));
+	}
+
+	serialize_byte(geomfile, minzoom, geompos, fname);
+
+	struct index index;
+	index.start = geomstart;
+	index.end = *geompos;
+	index.index = encode(bbox[0] / 2 + bbox[2] / 2, bbox[1] / 2 + bbox[3] / 2);
+	fwrite_check(&index, sizeof(struct index), 1, indexfile, fname);
+	*indexpos += sizeof(struct index);
+
+	for (i = 0; i < 2; i++) {
+		if (bbox[i] < file_bbox[i]) {
+			file_bbox[i] = bbox[i];
+		}
+	}
+	for (i = 2; i < 4; i++) {
+		if (bbox[i] > file_bbox[i]) {
+			file_bbox[i] = bbox[i];
+		}
+	}
+
+	if (*seq % 10000 == 0) {
+		if (!quiet) {
+			fprintf(stderr, "Read %.2f million features\r", *seq / 1000000.0);
+		}
+	}
+	(*seq)++;
+
+	return 1;
+}
+
+int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent, char *additional) {
 	int ret = EXIT_SUCCESS;
 
 	char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1];
@@ -533,8 +690,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
 				found_hashes++;
 
 				if (found_hashes == 50 && found_features == 0) {
-					fprintf(stderr, "%s:%d: Not finding any GeoJSON features in input. Is your file just bare geometries?\n", reading, jp->line);
+					fprintf(stderr, "%s:%d: Not finding any GeoJSON features in input after 50 objects. Is your file just bare geometries?\n", reading, jp->line);
-					break;
 				}
 			}
 
@@ -552,24 +708,6 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
 				continue;
 			}
 
-			json_object *geometry_type = json_hash_get(geometry, "type");
-			if (geometry_type == NULL) {
-				static int warned = 0;
-				if (!warned) {
-					fprintf(stderr, "%s:%d: null geometry (additional not reported)\n", reading, jp->line);
-					warned = 1;
-				}
-
-				json_free(j);
-				continue;
-			}
-
-			if (geometry_type->type != JSON_STRING) {
-				fprintf(stderr, "%s:%d: geometry without type\n", reading, jp->line);
-				json_free(j);
-				continue;
-			}
-
 			json_object *properties = json_hash_get(j, "properties");
 			if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
 				fprintf(stderr, "%s:%d: feature without properties hash\n", reading, jp->line);
@@ -577,142 +715,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
 				continue;
 			}
 
-			json_object *coordinates = json_hash_get(geometry, "coordinates");
+			json_object *geometries = json_hash_get(geometry, "geometries");
-			if (coordinates == NULL || coordinates->type != JSON_ARRAY) {
+			if (geometries != NULL) {
-				fprintf(stderr, "%s:%d: feature without coordinates array\n", reading, jp->line);
+				int g;
-				json_free(j);
+				for (g = 0; g < geometries->length; g++) {
-				continue;
+					serialize_geometry(geometries->array[g], properties, reading, jp, &seq, &metapos, &geompos, &indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, n, droprate, file_bbox);
-			}
-
-			int t;
-			for (t = 0; t < GEOM_TYPES; t++) {
-				if (strcmp(geometry_type->string, geometry_names[t]) == 0) {
-					break;
 				}
-			}
+			} else {
-			if (t >= GEOM_TYPES) {
+				serialize_geometry(geometry, properties, reading, jp, &seq, &metapos, &geompos, &indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, maxzoom, n, droprate, file_bbox);
-				fprintf(stderr, "%s:%d: Can't handle geometry type %s\n", reading, jp->line, geometry_type->string);
-				json_free(j);
-				continue;
-			}
-
-			{
-				unsigned bbox[] = {UINT_MAX, UINT_MAX, 0, 0};
-
-				int nprop = 0;
-				if (properties->type == JSON_HASH) {
-					nprop = properties->length;
-				}
-
-				long long metastart = metapos;
-				char *metakey[nprop];
-				char *metaval[nprop];
-				int metatype[nprop];
-				int m = 0;
-
-				int i;
-				for (i = 0; i < nprop; i++) {
-					if (properties->keys[i]->type == JSON_STRING) {
-						if (exclude_all) {
-							if (!is_pooled(include, properties->keys[i]->string, VT_STRING)) {
-								continue;
-							}
-						} else if (is_pooled(exclude, properties->keys[i]->string, VT_STRING)) {
-							continue;
-						}
-
-						metakey[m] = properties->keys[i]->string;
-
-						if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) {
-							metatype[m] = VT_STRING;
-							metaval[m] = properties->values[i]->string;
-							m++;
-						} else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) {
-							metatype[m] = VT_NUMBER;
-							metaval[m] = properties->values[i]->string;
-							m++;
-						} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) {
-							metatype[m] = VT_BOOLEAN;
-							metaval[m] = properties->values[i]->type == JSON_TRUE ? "true" : "false";
-							m++;
-						} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) {
-							;
-						} else {
-							fprintf(stderr, "%s:%d: Unsupported property type for %s\n", reading, jp->line, properties->keys[i]->string);
-							json_free(j);
-							continue;
-						}
-					}
-				}
-
-				serialize_int(metafile, m, &metapos, fname);
-				for (i = 0; i < m; i++) {
-					serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), &metapos, fname);
-					serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), &metapos, fname);
-				}
-
-				long long geomstart = geompos;
-
-				serialize_byte(geomfile, mb_geometry[t], &geompos, fname);
-				serialize_byte(geomfile, n, &geompos, fname);
-				serialize_long_long(geomfile, metastart, &geompos, fname);
-				long long wx = initial_x, wy = initial_y;
-				parse_geometry(t, coordinates, bbox, &geompos, geomfile, VT_MOVETO, fname, jp, &wx, &wy, &initialized);
-				serialize_byte(geomfile, VT_END, &geompos, fname);
-
-				/*
-				 * Note that minzoom for lines is the dimension
-				 * of the geometry in world coordinates, but
-				 * for points is the lowest zoom level (in tiles,
-				 * not in pixels) at which it should be drawn.
-				 *
-				 * So a line that is too small for, say, z8
-				 * will have minzoom of 18 (if tile detail is 10),
-				 * not 8.
-				 */
-				int minzoom = 0;
-				if (mb_geometry[t] == VT_LINE) {
-					for (minzoom = 0; minzoom < 31; minzoom++) {
-						unsigned mask = 1 << (32 - (minzoom + 1));
-
-						if (((bbox[0] & mask) != (bbox[2] & mask)) || ((bbox[1] & mask) != (bbox[3] & mask))) {
-							break;
-						}
-					}
-				} else if (mb_geometry[t] == VT_POINT) {
-					double r = ((double) rand()) / RAND_MAX;
-					if (r == 0) {
-						r = .00000001;
-					}
-					minzoom = maxzoom - floor(log(r) / -log(droprate));
-				}
-
-				serialize_byte(geomfile, minzoom, &geompos, fname);
-
-				struct index index;
-				index.start = geomstart;
-				index.end = geompos;
-				index.index = encode(bbox[0] / 2 + bbox[2] / 2, bbox[1] / 2 + bbox[3] / 2);
-				fwrite_check(&index, sizeof(struct index), 1, indexfile, fname);
-				indexpos += sizeof(struct index);
-
-				for (i = 0; i < 2; i++) {
-					if (bbox[i] < file_bbox[i]) {
-						file_bbox[i] = bbox[i];
-					}
-				}
-				for (i = 2; i < 4; i++) {
-					if (bbox[i] > file_bbox[i]) {
-						file_bbox[i] = bbox[i];
-					}
-				}
-
-				if (seq % 10000 == 0) {
-					if (!quiet) {
-						fprintf(stderr, "Read %.2f million features\r", seq / 1000000.0);
-					}
-				}
-				seq++;
 			}
 
 			json_free(j);
@@ -991,7 +1001,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
 		fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", seq, (long long) geomst.st_size, (long long) metast.st_size, poolfile->off);
 	}
 
-	int written = traverse_zooms(fd, size, meta, stringpool, file_bbox, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, full_detail, low_detail, min_detail);
+	int written = traverse_zooms(fd, size, meta, stringpool, file_bbox, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail);
 
 	if (maxzoom != written) {
 		fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written);
@@ -1044,6 +1054,10 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
 }
 
 int main(int argc, char **argv) {
+#ifdef MTRACE
+	mtrace();
+#endif
+
 	extern int optind;
 	extern char *optarg;
 	int i;
@@ -1059,6 +1073,7 @@ int main(int argc, char **argv) {
 	int buffer = 5;
 	const char *tmpdir = "/tmp";
 	char prevent[256];
+	char additional[256];
 
 	struct pool exclude, include;
 	pool_init(&exclude, 0);
@@ -1067,9 +1082,10 @@ int main(int argc, char **argv) {
 
 	for (i = 0; i < 256; i++) {
 		prevent[i] = 0;
+		additional[i] = 0;
 	}
 
-	while ((i = getopt(argc, argv, "l:n:z:Z:d:D:m:o:x:y:r:b:fXt:g:p:vq")) != -1) {
+	while ((i = getopt(argc, argv, "l:n:z:Z:d:D:m:o:x:y:r:b:fXt:g:p:vqa:")) != -1) {
 		switch (i) {
 		case 'n':
 			name = optarg;
@@ -1147,12 +1163,19 @@ int main(int argc, char **argv) {
 			}
 		} break;
 
+		case 'a': {
+			char *cp;
+			for (cp = optarg; *cp != '\0'; cp++) {
+				additional[*cp & 0xFF] = 1;
+			}
+		} break;
+
 		case 'v':
 			fprintf(stderr, VERSION);
 			exit(EXIT_FAILURE);
 
 		default:
-			fprintf(stderr, "Usage: %s -o out.mbtiles [-n name] [-l layername] [-z maxzoom] [-Z minzoom] [-d detail] [-D lower-detail] [-m min-detail] [-x excluded-field ...] [-y included-field ...] [-X] [-r droprate] [-b buffer] [-t tmpdir] [-p rcfs] [file.json ...]\n", argv[0]);
+			fprintf(stderr, "Usage: %s -o out.mbtiles [-n name] [-l layername] [-z maxzoom] [-Z minzoom] [-d detail] [-D lower-detail] [-m min-detail] [-x excluded-field ...] [-y included-field ...] [-X] [-r droprate] [-b buffer] [-t tmpdir] [-a rco] [-p sfkld] [-q] [file.json ...]\n", argv[0]);
 			exit(EXIT_FAILURE);
 		}
 	}
@@ -1163,8 +1186,13 @@ int main(int argc, char **argv) {
 	}
 
 	if (full_detail <= 0) {
-		// ~0.5m accuracy at whatever zoom
+		if (maxzoom >= 14) {
-		// 12 bits (4096 units) at z14
+			// ~0.5m accuracy at z14
+			full_detail = 12;
+		} else {
+			// as good as we can get without breaking GL
+			full_detail = 13;
+		}
 
 		full_detail = 26 - maxzoom;
 	}
@@ -1188,8 +1216,13 @@ int main(int argc, char **argv) {
 	sqlite3 *outdb = mbtiles_open(outdir, argv);
 	int ret = EXIT_SUCCESS;
 
-	ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent);
+	ret = read_json(argc - optind, argv + optind, name ? name : outdir, layer, maxzoom, minzoom, outdb, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, prevent, additional);
 
 	mbtiles_close(outdb, argv);
+
+#ifdef MTRACE
+	muntrace();
+#endif
+
 	return ret;
 }

man/tippecanoe.1

@@ -64,6 +64,8 @@ it encounters.
 \-o \fIfile\fP\&.mbtiles: Name the output file.
 .IP \(bu 2
 \-f: Delete the mbtiles file if it already exists instead of giving an error
+.IP \(bu 2
+\-t \fIdirectory\fP: Put the temporary files in \fIdirectory\fP\&.
 .RE
 .SS Zoom levels and resolution
 .RS
@@ -72,7 +74,7 @@ it encounters.
 .IP \(bu 2
 \-Z \fIzoom\fP: Lowest (minzoom) zoom level (default 0)
 .IP \(bu 2
-\-d \fIdetail\fP: Detail at base zoom level (default 26\-basezoom, ~0.5m, for tile resolution of 4096 if \-z14)
+\-d \fIdetail\fP: Detail at base zoom level (default 12 at \-z14 or higher, or 13 at \-z13 or lower. Detail beyond 13 has rendering problems with Mapbox GL.)
 .IP \(bu 2
 \-D \fIdetail\fP: Detail at lower zoom levels (default 10, for tile resolution of 1024)
 .IP \(bu 2
@@ -96,23 +98,26 @@ it encounters.
 .IP \(bu 2
 \-g \fIgamma\fP: Rate at which especially dense dots are dropped (default 0, for no effect). A gamma of 2 reduces the number of dots less than a pixel apart to the square root of their original number.
 .RE
+.SS Doing more
+.RS
+.IP \(bu 2
+\-ac: Coalesce adjacent line and polygon features that have the same properties
+.IP \(bu 2
+\-ar: Try reversing the directions of lines to make them coalesce and compress better
+.IP \(bu 2
+\-ao: Reorder features to put ones with the same properties in sequence, to try to get them to coalesce
+.IP \(bu 2
+\-al: Let "dot" dropping at lower zooms apply to lines too
+.RE
 .SS Doing less
 .RS
 .IP \(bu 2
 \-ps: Don't simplify lines
 .IP \(bu 2
-\-pr: Don't reverse the direction of lines to make them coalesce better
-.IP \(bu 2
-\-pc: Don't coalesce features with the same properties
-.IP \(bu 2
 \-pf: Don't limit tiles to 200,000 features
 .IP \(bu 2
 \-pk: Don't limit tiles to 500K bytes
 .IP \(bu 2
-\-po: Don't reorder features to put the same properties in sequence
-.IP \(bu 2
-\-pl: Let "dot" simplification apply to lines too
-.IP \(bu 2
 \-pd: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries.
 .IP \(bu 2
 \-q: Work quietly instead of reporting progress
@@ -167,7 +172,7 @@ I don't know why 2.5 is the appropriate number, but the densities of many differ
 data sets fall off at about this same rate. You can use \-r to specify a different rate.
 .PP
 You can use the gamma option to thin out especially dense clusters of points.
-For any area that where dots are closer than one pixel together (at whatever zoom level),
+For any area where dots are closer than one pixel together (at whatever zoom level),
 a gamma of 3, for example, will reduce these clusters to the cube root of their original density.
 .PP
 For line features, it drops any features that are too small to draw at all.
@@ -187,10 +192,28 @@ If a tile is larger than 500K, it will try encoding that tile at progressively
 lower resolutions before failing if it still doesn't fit.
 .SH Development
 .PP
-Requires protoc (\fB\fCbrew install protobuf\fR or
+Requires protoc and sqlite3. Rebuilding the manpage
-\fB\fCapt\-get install libprotobuf\-dev\fR and \fB\fCprotobuf\-compiler\fR),
+uses md2man (\fB\fCgem install md2man\fR).
-\fB\fCmd2man\fR (\fB\fCgem install md2man\fR), and sqlite3 (\fB\fCapt\-get install libsqlite3\-dev\fR).
+.PP
-To build:
+MacOS:
+.PP
+.RS
+.nf
+brew install protobuf
+.fi
+.RE
+.PP
+Linux:
+.PP
+.RS
+.nf
+sudo apt\-get install libprotobuf\-dev
+sudo apt\-get install protobuf\-compiler
+sudo apt\-get install libsqlite3\-dev
+.fi
+.RE
+.PP
+Then build:
 .PP
 .RS
 .nf
@@ -212,4 +235,4 @@ Check out some examples of maps made with tippecanoe
 .SH Name
 .PP
 The name is a joking reference
-\[la]http://en.wikipedia.org/wiki/Tippecanoe_and_Tyler_Too\[ra] to making tiles.
+\[la]http://en.wikipedia.org/wiki/Tippecanoe_and_Tyler_Too\[ra] to a "tiler" for making map tiles.

pool.c

@@ -1,10 +1,11 @@
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include "pool.h"
 
 #define POOL_WIDTH 256
 
-static int hash(char *s) {
+static int hash(const char *s) {
 	int h = 0;
 	for (; *s; s++) {
 		h = h * 37 + *s;
@@ -13,7 +14,7 @@ static int hash(char *s) {
 	return h;
 }
 
-struct pool_val *pool(struct pool *p, char *s, int type) {
+struct pool_val *pool(struct pool *p, const char *s, int type) {
 	int h = hash(s);
 	struct pool_val **v = &(p->vals[h]);
 
@@ -34,6 +35,10 @@ struct pool_val *pool(struct pool *p, char *s, int type) {
 	}
 
 	*v = malloc(sizeof(struct pool_val));
+	if (*v == NULL) {
+		fprintf(stderr, "out of memory making string pool\n");
+		exit(EXIT_FAILURE);
+	}
 	(*v)->left = NULL;
 	(*v)->right = NULL;
 	(*v)->next = NULL;
@@ -52,7 +57,7 @@ struct pool_val *pool(struct pool *p, char *s, int type) {
 	return *v;
 }
 
-int is_pooled(struct pool *p, char *s, int type) {
+int is_pooled(struct pool *p, const char *s, int type) {
 	int h = hash(s);
 	struct pool_val **v = &(p->vals[h]);
 
@@ -78,7 +83,7 @@ int is_pooled(struct pool *p, char *s, int type) {
 void pool_free1(struct pool *p, void (*func)(void *)) {
 	while (p->head != NULL) {
 		if (func != NULL) {
-			func(p->head->s);
+			func((void *) p->head->s);
 		}
 
 		struct pool_val *next = p->head->next;
@@ -104,6 +109,10 @@ void pool_free_strings(struct pool *p) {
 void pool_init(struct pool *p, int n) {
 	p->n = n;
 	p->vals = calloc(POOL_WIDTH, sizeof(struct pool_val *));
+	if (p->vals == NULL) {
+		fprintf(stderr, "out of memory creating string pool\n");
+		exit(EXIT_FAILURE);
+	}
 	p->head = NULL;
 	p->tail = NULL;
 }

pool.h

@@ -1,5 +1,5 @@
 struct pool_val {
-	char *s;
+	const char *s;
 	int type;
 	int n;
 
@@ -17,8 +17,8 @@ struct pool {
 	int n;
 };
 
-struct pool_val *pool(struct pool *p, char *s, int type);
+struct pool_val *pool(struct pool *p, const char *s, int type);
 void pool_free(struct pool *p);
 void pool_free_strings(struct pool *p);
 void pool_init(struct pool *p, int n);
-int is_pooled(struct pool *p, char *s, int type);
+int is_pooled(struct pool *p, const char *s, int type);

tile-join.cc

@@ -0,0 +1,540 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sqlite3.h>
+#include <vector>
+#include <string>
+#include <map>
+#include <zlib.h>
+#include <math.h>
+#include "vector_tile.pb.h"
+#include "tile.h"
+
+extern "C" {
+#include "projection.h"
+#include "pool.h"
+#include "mbtiles.h"
+}
+
+std::string dequote(std::string s);
+
+struct stats {
+	int minzoom;
+	int maxzoom;
+	double midlat, midlon;
+	double minlat, minlon, maxlat, maxlon;
+};
+
+// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
+inline bool is_compressed(std::string const &data) {
+	return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B));
+}
+
+// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
+inline int decompress(std::string const &input, std::string &output) {
+	z_stream inflate_s;
+	inflate_s.zalloc = Z_NULL;
+	inflate_s.zfree = Z_NULL;
+	inflate_s.opaque = Z_NULL;
+	inflate_s.avail_in = 0;
+	inflate_s.next_in = Z_NULL;
+	if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) {
+		fprintf(stderr, "error: %s\n", inflate_s.msg);
+	}
+	inflate_s.next_in = (Bytef *) input.data();
+	inflate_s.avail_in = input.size();
+	size_t length = 0;
+	do {
+		output.resize(length + 2 * input.size());
+		inflate_s.avail_out = 2 * input.size();
+		inflate_s.next_out = (Bytef *) (output.data() + length);
+		int ret = inflate(&inflate_s, Z_FINISH);
+		if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
+			fprintf(stderr, "error: %s\n", inflate_s.msg);
+			return 0;
+		}
+
+		length += (2 * input.size() - inflate_s.avail_out);
+	} while (inflate_s.avail_out == 0);
+	inflateEnd(&inflate_s);
+	output.resize(length);
+	return 1;
+}
+
+// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
+static inline int compress(std::string const &input, std::string &output) {
+	z_stream deflate_s;
+	deflate_s.zalloc = Z_NULL;
+	deflate_s.zfree = Z_NULL;
+	deflate_s.opaque = Z_NULL;
+	deflate_s.avail_in = 0;
+	deflate_s.next_in = Z_NULL;
+	deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
+	deflate_s.next_in = (Bytef *) input.data();
+	deflate_s.avail_in = input.size();
+	size_t length = 0;
+	do {
+		size_t increase = input.size() / 2 + 1024;
+		output.resize(length + increase);
+		deflate_s.avail_out = increase;
+		deflate_s.next_out = (Bytef *) (output.data() + length);
+		int ret = deflate(&deflate_s, Z_FINISH);
+		if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
+			return -1;
+		}
+		length += (increase - deflate_s.avail_out);
+	} while (deflate_s.avail_out == 0);
+	deflateEnd(&deflate_s);
+	output.resize(length);
+	return 0;
+}
+
+void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, std::vector<std::string> &header, std::map<std::string, std::vector<std::string> > &mapping, struct pool *exclude, int ifmatched) {
+	GOOGLE_PROTOBUF_VERIFY_VERSION;
+
+	// https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp
+	mapnik::vector::tile tile;
+	mapnik::vector::tile outtile;
+
+	if (is_compressed(message)) {
+		std::string uncompressed;
+		decompress(message, uncompressed);
+		if (!tile.ParseFromString(uncompressed)) {
+			fprintf(stderr, "Couldn't decompress tile %d/%u/%u\n", z, x, y);
+			exit(EXIT_FAILURE);
+		}
+	} else if (!tile.ParseFromString(message)) {
+		fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y);
+		exit(EXIT_FAILURE);
+	}
+
+	for (int l = 0; l < tile.layers_size(); l++) {
+		mapnik::vector::tile_layer layer = tile.layers(l);
+		mapnik::vector::tile_layer *outlayer = outtile.add_layers();
+
+		outlayer->set_name(layer.name());
+		outlayer->set_version(layer.version());
+		outlayer->set_extent(layer.extent());
+
+		const char *ln = layer.name().c_str();
+
+		int ll;
+		for (ll = 0; ll < *nlayers; ll++) {
+			if (strcmp((*layernames)[ll], ln) == 0) {
+				break;
+			}
+		}
+		if (ll == *nlayers) {
+			*file_keys = (struct pool *) realloc(*file_keys, (ll + 1) * sizeof(struct pool));
+			*layernames = (char **) realloc(*layernames, (ll + 1) * sizeof(char *));
+
+			if (*file_keys == NULL) {
+				perror("realloc file_keys");
+				exit(EXIT_FAILURE);
+			}
+			if (*layernames == NULL) {
+				perror("realloc layernames");
+				exit(EXIT_FAILURE);
+			}
+
+			pool_init(&((*file_keys)[ll]), 0);
+			(*layernames)[ll] = strdup(ln);
+			*nlayers = ll + 1;
+		}
+
+		struct pool keys, values;
+		pool_init(&keys, 0);
+		pool_init(&values, 0);
+
+		for (int f = 0; f < layer.features_size(); f++) {
+			mapnik::vector::tile_feature feat = layer.features(f);
+			std::vector<int> feature_tags;
+			int matched = 0;
+
+			for (int t = 0; t + 1 < feat.tags_size(); t += 2) {
+				const char *key = layer.keys(feat.tags(t)).c_str();
+				mapnik::vector::tile_value const &val = layer.values(feat.tags(t + 1));
+				char *value;
+				int type = -1;
+
+				if (val.has_string_value()) {
+					value = strdup(val.string_value().c_str());
+					type = VT_STRING;
+				} else if (val.has_int_value()) {
+					if (asprintf(&value, "%lld", (long long) val.int_value()) >= 0) {
+						type = VT_NUMBER;
+					}
+				} else if (val.has_double_value()) {
+					if (asprintf(&value, "%g", val.double_value()) >= 0) {
+						type = VT_NUMBER;
+					}
+				} else if (val.has_float_value()) {
+					if (asprintf(&value, "%g", val.float_value()) >= 0) {
+						type = VT_NUMBER;
+					}
+				} else if (val.has_bool_value()) {
+					if (asprintf(&value, "%s", val.bool_value() ? "true" : "false") >= 0) {
+						type = VT_BOOLEAN;
+					}
+				} else if (val.has_sint_value()) {
+					if (asprintf(&value, "%lld", (long long) val.sint_value()) >= 0) {
+						type = VT_NUMBER;
+					}
+				} else if (val.has_uint_value()) {
+					if (asprintf(&value, "%llu", (long long) val.uint_value()) >= 0) {
+						type = VT_NUMBER;
+					}
+				} else {
+					continue;
+				}
+
+				if (type < 0) {
+					continue;
+				}
+
+				if (!is_pooled(exclude, key, VT_STRING)) {
+					if (!is_pooled(&((*file_keys)[ll]), key, type)) {
+						pool(&((*file_keys)[ll]), strdup(key), type);
+					}
+
+					struct pool_val *k, *v;
+
+					if (is_pooled(&keys, key, VT_STRING)) {
+						k = pool(&keys, key, VT_STRING);
+					} else {
+						k = pool(&keys, strdup(key), VT_STRING);
+					}
+
+					if (is_pooled(&values, value, type)) {
+						v = pool(&values, value, type);
+					} else {
+						v = pool(&values, strdup(value), type);
+					}
+
+					feature_tags.push_back(k->n);
+					feature_tags.push_back(v->n);
+				}
+
+				if (strcmp(key, header[0].c_str()) == 0) {
+					std::map<std::string, std::vector<std::string> >::iterator ii = mapping.find(std::string(value));
+
+					if (ii != mapping.end()) {
+						std::vector<std::string> fields = ii->second;
+						matched = 1;
+
+						for (unsigned i = 1; i < fields.size(); i++) {
+							std::string joinkey = header[i];
+							std::string joinval = fields[i];
+							int type = VT_STRING;
+
+							if (joinval.size() > 0) {
+								if (joinval[0] == '"') {
+									joinval = dequote(joinval);
+								} else if ((joinval[0] >= '0' && joinval[0] <= '9') || joinval[0] == '-') {
+									type = VT_NUMBER;
+								}
+							}
+
+							const char *sjoinkey = joinkey.c_str();
+							const char *sjoinval = joinval.c_str();
+
+							if (!is_pooled(exclude, sjoinkey, VT_STRING)) {
+								if (!is_pooled(&((*file_keys)[ll]), sjoinkey, type)) {
+									pool(&((*file_keys)[ll]), strdup(sjoinkey), type);
+								}
+
+								struct pool_val *k, *v;
+
+								if (is_pooled(&keys, sjoinkey, VT_STRING)) {
+									k = pool(&keys, sjoinkey, VT_STRING);
+								} else {
+									k = pool(&keys, strdup(sjoinkey), VT_STRING);
+								}
+
+								if (is_pooled(&values, sjoinval, type)) {
+									v = pool(&values, sjoinval, type);
+								} else {
+									v = pool(&values, strdup(sjoinval), type);
+								}
+
+								feature_tags.push_back(k->n);
+								feature_tags.push_back(v->n);
+							}
+						}
+					}
+				}
+
+				free(value);
+			}
+
+			if (matched || !ifmatched) {
+				mapnik::vector::tile_feature *outfeature = outlayer->add_features();
+				outfeature->set_type(feat.type());
+
+				for (int g = 0; g < feat.geometry_size(); g++) {
+					outfeature->add_geometry(feat.geometry(g));
+				}
+
+				for (unsigned i = 0; i < feature_tags.size(); i++) {
+					outfeature->add_tags(feature_tags[i]);
+				}
+			}
+		}
+
+		struct pool_val *pv;
+		for (pv = keys.head; pv != NULL; pv = pv->next) {
+			outlayer->add_keys(pv->s, strlen(pv->s));
+		}
+		for (pv = values.head; pv != NULL; pv = pv->next) {
+			mapnik::vector::tile_value *tv = outlayer->add_values();
+
+			if (pv->type == VT_NUMBER) {
+				tv->set_double_value(atof(pv->s));
+			} else if (pv->type == VT_BOOLEAN) {
+				tv->set_bool_value(pv->s[0] == 't');
+			} else {
+				tv->set_string_value(pv->s);
+			}
+		}
+
+		pool_free_strings(&keys);
+		pool_free_strings(&values);
+	}
+
+	std::string s;
+	std::string compressed;
+
+	outtile.SerializeToString(&s);
+	compress(s, compressed);
+
+	if (compressed.size() > 500000) {
+		fprintf(stderr, "Tile %d/%u/%u size is %lld, >500000. Skipping this tile\n.", z, x, y, (long long) compressed.size());
+		return;
+	}
+
+	mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size());
+}
+
+void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string> > &mapping, struct pool *exclude, int ifmatched) {
+	sqlite3 *db;
+
+	if (sqlite3_open(fname, &db) != SQLITE_OK) {
+		fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db));
+		exit(EXIT_FAILURE);
+	}
+
+	const char *sql = "SELECT zoom_level, tile_column, tile_row, tile_data from tiles;";
+	sqlite3_stmt *stmt;
+	if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
+		fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
+		exit(EXIT_FAILURE);
+	}
+
+	while (sqlite3_step(stmt) == SQLITE_ROW) {
+		long long zoom = sqlite3_column_int(stmt, 0);
+		long long x = sqlite3_column_int(stmt, 1);
+		long long y = sqlite3_column_int(stmt, 2);
+		y = (1LL << zoom) - 1 - y;
+
+		int len = sqlite3_column_bytes(stmt, 3);
+		const char *s = (const char *) sqlite3_column_blob(stmt, 3);
+
+		fprintf(stderr, "%lld/%lld/%lld   \r", zoom, x, y);
+
+		handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb, header, mapping, exclude, ifmatched);
+	}
+
+	sqlite3_finalize(stmt);
+
+	if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'minzoom'", -1, &stmt, NULL) == SQLITE_OK) {
+		if (sqlite3_step(stmt) == SQLITE_ROW) {
+			st->minzoom = sqlite3_column_int(stmt, 0);
+		}
+		sqlite3_finalize(stmt);
+	}
+	if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'maxzoom'", -1, &stmt, NULL) == SQLITE_OK) {
+		if (sqlite3_step(stmt) == SQLITE_ROW) {
+			st->maxzoom = sqlite3_column_int(stmt, 0);
+		}
+		sqlite3_finalize(stmt);
+	}
+	if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'center'", -1, &stmt, NULL) == SQLITE_OK) {
+		if (sqlite3_step(stmt) == SQLITE_ROW) {
+			const unsigned char *s = sqlite3_column_text(stmt, 0);
+			sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat);
+		}
+		sqlite3_finalize(stmt);
+	}
+	if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'bounds'", -1, &stmt, NULL) == SQLITE_OK) {
+		if (sqlite3_step(stmt) == SQLITE_ROW) {
+			const unsigned char *s = sqlite3_column_text(stmt, 0);
+			sscanf((char *) s, "%lf,%lf,%lf,%lf", &st->minlon, &st->minlat, &st->maxlon, &st->maxlat);
+		}
+		sqlite3_finalize(stmt);
+	}
+
+	if (sqlite3_close(db) != SQLITE_OK) {
+		fprintf(stderr, "%s: could not close database: %s\n", fname, sqlite3_errmsg(db));
+		exit(EXIT_FAILURE);
+	}
+}
+
+void usage(char **argv) {
+	fprintf(stderr, "Usage: %s [-f] [-i] [-c joins.csv] [-x exclude ...] -o new.mbtiles source.mbtiles\n", argv[0]);
+	exit(EXIT_FAILURE);
+}
+
+#define MAXLINE 10000 /* XXX */
+
+std::vector<std::string> split(char *s) {
+	std::vector<std::string> ret;
+
+	while (*s && *s != '\n') {
+		char *start = s;
+		int within = 0;
+
+		for (; *s && *s != '\n'; s++) {
+			if (*s == '"') {
+				within = !within;
+			}
+
+			if (*s == ',' && !within) {
+				break;
+			}
+		}
+
+		std::string v = std::string(start, s - start);
+		ret.push_back(v);
+
+		if (*s == ',') {
+			s++;
+		}
+	}
+
+	return ret;
+}
+
+std::string dequote(std::string s) {
+	std::string out;
+	unsigned i;
+	for (i = 0; i < s.size(); i++) {
+		if (s[i] == '"') {
+			if (i + 1 < s.size() && s[i + 1] == '"') {
+				out.push_back('"');
+			}
+		} else {
+			out.push_back(s[i]);
+		}
+	}
+	return out;
+}
+
+void readcsv(char *fn, std::vector<std::string> &header, std::map<std::string, std::vector<std::string> > &mapping) {
+	FILE *f = fopen(fn, "r");
+	if (f == NULL) {
+		perror(fn);
+		exit(EXIT_FAILURE);
+	}
+
+	char s[MAXLINE];
+	if (fgets(s, MAXLINE, f)) {
+		header = split(s);
+
+		for (unsigned i = 0; i < header.size(); i++) {
+			header[i] = dequote(header[i]);
+		}
+	}
+	while (fgets(s, MAXLINE, f)) {
+		std::vector<std::string> line = split(s);
+		if (line.size() > 0) {
+			line[0] = dequote(line[0]);
+		}
+
+		for (unsigned i = 0; i < line.size() && i < header.size(); i++) {
+			// printf("putting %s\n", line[0].c_str());
+			mapping.insert(std::pair<std::string, std::vector<std::string> >(line[0], line));
+		}
+	}
+
+	fclose(f);
+}
+
+int main(int argc, char **argv) {
+	char *outfile = NULL;
+	char *csv = NULL;
+	int force = 0;
+	int ifmatched = 0;
+
+	std::vector<std::string> header;
+	std::map<std::string, std::vector<std::string> > mapping;
+
+	struct pool exclude;
+	pool_init(&exclude, 0);
+
+	extern int optind;
+	extern char *optarg;
+	int i;
+
+	while ((i = getopt(argc, argv, "fo:c:x:i")) != -1) {
+		switch (i) {
+		case 'o':
+			outfile = optarg;
+			break;
+
+		case 'f':
+			force = 1;
+			break;
+
+		case 'i':
+			ifmatched = 1;
+			break;
+
+		case 'c':
+			if (csv != NULL) {
+				fprintf(stderr, "Only one -c for now\n");
+				exit(EXIT_FAILURE);
+			}
+
+			csv = optarg;
+			readcsv(csv, header, mapping);
+			break;
+
+		case 'x':
+			pool(&exclude, optarg, VT_STRING);
+			break;
+
+		default:
+			usage(argv);
+		}
+	}
+
+	if (argc - optind != 1 || outfile == NULL) {
+		usage(argv);
+	}
+
+	if (force) {
+		unlink(outfile);
+	}
+
+	sqlite3 *outdb = mbtiles_open(outfile, argv);
+	struct stats st;
+	memset(&st, 0, sizeof(st));
+
+	struct pool *file_keys = NULL;
+	char **layernames = NULL;
+	int nlayers = 0;
+
+	for (i = optind; i < argc; i++) {
+		decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping, &exclude, ifmatched);
+	}
+
+	struct pool *fk[nlayers];
+	for (i = 0; i < nlayers; i++) {
+		fk[i] = &(file_keys[i]);
+	}
+
+	mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, fk, nlayers);
+	mbtiles_close(outdb, argv);
+
+	return 0;
+}

tile.cc

@@ -8,6 +8,7 @@
 #include <algorithm>
 #include <stdio.h>
 #include <unistd.h>
+#include <limits.h>
 #include <zlib.h>
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -27,6 +28,9 @@ extern "C" {
 
 #define CMD_BITS 3
 
+#define XSTRINGIFY(s) STRINGIFY(s)
+#define STRINGIFY(s) #s
+
 // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
 static inline int compress(std::string const &input, std::string &output) {
 	z_stream deflate_s;
@@ -358,7 +362,7 @@ void evaluate(std::vector<coalesce> &features, char *metabase, struct pool *file
 }
 #endif
 
-void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, signed char layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment) {
+void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment) {
 	if (geom.size() > 0 && nextzoom <= file_maxzoom) {
 		int xo, yo;
 		int span = 1 << (nextzoom - z);
@@ -429,7 +433,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb
 
 					// printf("type %d, meta %lld\n", t, metastart);
 					serialize_byte(geomfile[j], t, &geompos[j], fname);
-					serialize_byte(geomfile[j], layer, &geompos[j], fname);
+					serialize_long_long(geomfile[j], layer, &geompos[j], fname);
 					serialize_long_long(geomfile[j], metastart, &geompos[j], fname);
 					long long wx = initial_x, wy = initial_y;
 
@@ -452,7 +456,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb
 	}
 }
 
-long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, int child_shards) {
+long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional, int child_shards) {
 	int line_detail;
 	static bool evaluated = false;
 	double oprogress = 0;
@@ -532,8 +536,8 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 				break;
 			}
 
-			signed char layer;
+			long long layer;
-			deserialize_byte(geoms, &layer);
+			deserialize_long_long(geoms, &layer);
 
 			long long metastart;
 			deserialize_long_long(geoms, &metastart);
@@ -594,7 +598,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 				continue;
 			}
 
-			if (gamma >= 0 && (t == VT_POINT || (prevent['l' & 0xFF] && t == VT_LINE))) {
+			if (gamma >= 0 && (t == VT_POINT || (additional['l' & 0xFF] && t == VT_LINE))) {
 				seq++;
 				if (seq >= 0) {
 					seq -= interval;
@@ -659,7 +663,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 			}
 #endif
 
-			if (t == VT_LINE && !prevent['r' & 0xFF]) {
+			if (t == VT_LINE && additional['r' & 0xFF]) {
 				geom = reorder_lines(geom);
 			}
 
@@ -701,7 +705,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 		}
 
 		for (j = 0; j < nlayers; j++) {
-			if (!prevent['o' & 0xFF]) {
+			if (additional['o' & 0xFF]) {
 				std::sort(features[j].begin(), features[j].end());
 			}
 
@@ -716,7 +720,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 				}
 #endif
 
-				if (!prevent['c' & 0xFF] && out.size() > 0 && out[y].geom.size() + features[j][x].geom.size() < 20000 && coalcmp(&features[j][x], &out[y]) == 0 && features[j][x].type != VT_POINT) {
+				if (additional['c' & 0xFF] && out.size() > 0 && out[y].geom.size() + features[j][x].geom.size() < 20000 && coalcmp(&features[j][x], &out[y]) == 0 && features[j][x].type != VT_POINT) {
 					unsigned z;
 					for (z = 0; z < features[j][x].geom.size(); z++) {
 						out[y].geom.push_back(features[j][x].geom[z]);
@@ -794,6 +798,12 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 				return count;
 			}
 		} else {
+			int i;
+			for (i = 0; i < nlayers; i++) {
+				pool_free(&keys1[i]);
+				pool_free(&values1[i]);
+			}
+
 			return count;
 		}
 	}
@@ -828,6 +838,7 @@ struct write_tile_args {
 	double gamma;
 	int nlayers;
 	char *prevent;
+	char *additional;
 	int child_shards;
 	int *geomfd;
 	off_t *geom_size;
@@ -875,7 +886,7 @@ void run_thread(write_tile_args *arg) {
 
 			// fprintf(stderr, "%d/%u/%u\n", z, x, y);
 
-			long long len = write_tile(&geom, arg->metabase, arg->stringpool, arg->file_bbox, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->maxzoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, *arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->child_shards);
+			long long len = write_tile(&geom, arg->metabase, arg->stringpool, arg->file_bbox, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->maxzoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, *arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards);
 
 			if (len < 0) {
 				return; // XXX how to report errors from threads?
@@ -896,7 +907,7 @@ void run_thread(write_tile_args *arg) {
 	}
 }
 
-int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) {
+int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail) {
 	int i;
 	for (i = 0; i <= maxzoom; i++) {
 		long long most = 0;
@@ -905,7 +916,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
 		int subfd[TEMP_FILES];
 		int j;
 		for (j = 0; j < TEMP_FILES; j++) {
-			char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1];
+			char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX" XSTRINGIFY(INT_MAX)) + 1];
 			sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j);
 			subfd[j] = mkstemp(geomname);
 			// printf("%s\n", geomname);
@@ -1013,6 +1024,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
 			args.gamma = gamma;
 			args.nlayers = nlayers;
 			args.prevent = prevent;
+			args.additional = additional;
 			args.child_shards = TEMP_FILES / threads;
 
 			args.geomfd = geomfd;

tile.h

@@ -25,9 +25,9 @@ void deserialize_uint(char **f, unsigned *n);
 void deserialize_byte(char **f, signed char *n);
 struct pool_val *deserialize_string(char **f, struct pool *p, int type);
 
-long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent);
+long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional);
 
-int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail);
+int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail);
 
 extern unsigned initial_x, initial_y;
 extern int geometry_scale;