Merge pull request #65 from mapbox/skip

Skipping over zoom levels when there is a minzoom, and other performance improvements

geojson.c

@@ -75,11 +75,17 @@ void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fn
 		unsigned char b = zigzag & 0x7F;
 		if ((zigzag >> 7) != 0) {
 			b |= 0x80;
-			fwrite_check(&b, sizeof(unsigned char), 1, out, fname);
+			if (putc(b, out) == EOF) {
+				fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
+				exit(EXIT_FAILURE);
+			}
 			*fpos += 1;
 			zigzag >>= 7;
 		} else {
-			fwrite_check(&b, sizeof(unsigned char), 1, out, fname);
+			if (putc(b, out) == EOF) {
+				fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
+				exit(EXIT_FAILURE);
+			}
 			*fpos += 1;
 			break;
 		}
@@ -954,14 +960,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
 		exit(EXIT_FAILURE);
 	}
 
-	int fd[4];
+	int fd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)];
-	off_t size[4];
+	off_t size[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)];
 
 	fd[0] = geomfd;
 	size[0] = geomst.st_size;
 
 	int j;
-	for (j = 1; j < 4; j++) {
+	for (j = 1; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
 		fd[j] = -1;
 		size[j] = 0;
 	}

geometry.cc

@@ -84,7 +84,7 @@ void to_tile_scale(drawvec &geom, int z, int detail) {
 	}
 }
 
-drawvec remove_noop(drawvec geom, int type) {
+drawvec remove_noop(drawvec geom, int type, int shift) {
 	// first pass: remove empty linetos
 
 	long long x = 0, y = 0;
@@ -92,7 +92,7 @@ drawvec remove_noop(drawvec geom, int type) {
 	unsigned i;
 
 	for (i = 0; i < geom.size(); i++) {
-		if (geom[i].op == VT_LINETO && geom[i].x == x && geom[i].y == y) {
+		if (geom[i].op == VT_LINETO && (geom[i].x >> shift) == x && (geom[i].y >> shift) == y) {
 			continue;
 		}
 
@@ -100,8 +100,8 @@ drawvec remove_noop(drawvec geom, int type) {
 			out.push_back(geom[i]);
 		} else { /* moveto or lineto */
 			out.push_back(geom[i]);
-			x = geom[i].x;
+			x = geom[i].x >> shift;
-			y = geom[i].y;
+			y = geom[i].y >> shift;
 		}
 	}
 
@@ -137,7 +137,7 @@ drawvec remove_noop(drawvec geom, int type) {
 
 		for (i = 0; i < geom.size(); i++) {
 			if (geom[i].op == VT_MOVETO) {
-				if (i > 0 && geom[i - 1].op == VT_LINETO && geom[i - 1].x == geom[i].x && geom[i - 1].y == geom[i].y) {
+				if (i > 0 && geom[i - 1].op == VT_LINETO && (geom[i - 1].x >> shift) == (geom[i].x >> shift) && (geom[i - 1].y >> shift) == (geom[i].y >> shift)) {
 					continue;
 				}
 			}

geometry.hh

@@ -18,7 +18,7 @@ typedef std::vector<draw> drawvec;
 
 drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox);
 void to_tile_scale(drawvec &geom, int z, int detail);
-drawvec remove_noop(drawvec geom, int type);
+drawvec remove_noop(drawvec geom, int type, int shift);
 drawvec clip_point(drawvec &geom, int z, int detail, long long buffer);
 drawvec clip_poly(drawvec &geom, int z, int detail, int buffer);
 drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double *accum_area);

jsonpull.c

@@ -499,7 +499,6 @@ again:
 
 		json_object *s = add_object(j, JSON_STRING);
 		if (s != NULL) {
-			val.buf = realloc(val.buf, val.n + 1);
 			s->string = val.buf;
 			s->length = val.n;
 		} else {

mbtiles.c

@@ -228,11 +228,7 @@ void mbtiles_close(sqlite3 *outdb, char **argv) {
 	char *err;
 
 	if (sqlite3_exec(outdb, "ANALYZE;", NULL, NULL, &err) != SQLITE_OK) {
-		fprintf(stderr, "%s: index metadata: %s\n", argv[0], err);
+		fprintf(stderr, "%s: ANALYZE failed: %s\n", argv[0], err);
-		exit(EXIT_FAILURE);
-	}
-	if (sqlite3_exec(outdb, "VACUUM;", NULL, NULL, &err) != SQLITE_OK) {
-		fprintf(stderr, "%s: index tiles: %s\n", argv[0], err);
 		exit(EXIT_FAILURE);
 	}
 	if (sqlite3_close(outdb) != SQLITE_OK) {

pool.c

@@ -2,11 +2,23 @@
 #include <string.h>
 #include "pool.h"
 
-static struct pool_val *pool1(struct pool *p, char *s, int type, int (*compare)(const char *, const char *)) {
+#define POOL_WIDTH 256
-	struct pool_val **v = &(p->vals);
+
+static int hash(char *s) {
+	int h = 0;
+	for (; *s; s++) {
+		h = h * 37 + *s;
+	}
+	h = h & 0xFF;
+	return h;
+}
+
+struct pool_val *pool(struct pool *p, char *s, int type) {
+	int h = hash(s);
+	struct pool_val **v = &(p->vals[h]);
 
 	while (*v != NULL) {
-		int cmp = compare(s, (*v)->s);
+		int cmp = strcmp(s, (*v)->s);
 
 		if (cmp == 0) {
 			cmp = type - (*v)->type;
@@ -41,7 +53,8 @@ static struct pool_val *pool1(struct pool *p, char *s, int type, int (*compare)(
 }
 
 int is_pooled(struct pool *p, char *s, int type) {
-	struct pool_val **v = &(p->vals);
+	int h = hash(s);
+	struct pool_val **v = &(p->vals[h]);
 
 	while (*v != NULL) {
 		int cmp = strcmp(s, (*v)->s);
@@ -62,10 +75,6 @@ int is_pooled(struct pool *p, char *s, int type) {
 	return 0;
 }
 
-struct pool_val *pool(struct pool *p, char *s, int type) {
-	return pool1(p, s, type, strcmp);
-}
-
 void pool_free1(struct pool *p, void (*func)(void *)) {
 	while (p->head != NULL) {
 		if (func != NULL) {
@@ -79,6 +88,8 @@ void pool_free1(struct pool *p, void (*func)(void *)) {
 
 	p->head = NULL;
 	p->tail = NULL;
+
+	free(p->vals);
 	p->vals = NULL;
 }
 
@@ -92,7 +103,7 @@ void pool_free_strings(struct pool *p) {
 
 void pool_init(struct pool *p, int n) {
 	p->n = n;
-	p->vals = NULL;
+	p->vals = calloc(POOL_WIDTH, sizeof(struct pool_val *));
 	p->head = NULL;
 	p->tail = NULL;
 }

pool.h

@@ -10,7 +10,7 @@ struct pool_val {
 };
 
 struct pool {
-	struct pool_val *vals;
+	struct pool_val **vals;
 
 	struct pool_val *head;
 	struct pool_val *tail;

tile.cc

@@ -236,7 +236,7 @@ mapnik::vector::tile create_tile(char **layernames, int line_detail, std::vector
 		unsigned x;
 		for (x = 0; x < features[i].size(); x++) {
 			if (features[i][x].type == VT_LINE || features[i][x].type == VT_POLYGON) {
-				features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type);
+				features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type, 0);
 			}
 
 			mapnik::vector::tile_feature *feature = layer->add_features();
@@ -358,7 +358,96 @@ void evaluate(std::vector<coalesce> &features, char *metabase, struct pool *file
 }
 #endif
 
-long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) {
+void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, signed char layer, long long metastart, signed char feature_minzoom) {
+	if (geom.size() > 0 && nextzoom <= file_maxzoom) {
+		int xo, yo;
+		int span = 1 << (nextzoom - z);
+
+		// Get the feature bounding box in pixel (256) coordinates at the child zoom
+		// in order to calculate which sub-tiles it can touch including the buffer.
+		long long bbox2[4];
+		int k;
+		for (k = 0; k < 4; k++) {
+			// Division instead of right-shift because coordinates can be negative
+			bbox2[k] = bbox[k] / (1 << (32 - nextzoom - 8));
+		}
+		bbox2[0] -= buffer;
+		bbox2[1] -= buffer;
+		bbox2[2] += buffer;
+		bbox2[3] += buffer;
+
+		for (k = 0; k < 4; k++) {
+			if (bbox2[k] < 0) {
+				bbox2[k] = 0;
+			}
+			if (bbox2[k] >= 256 * span) {
+				bbox2[k] = 256 * (span - 1);
+			}
+
+			bbox2[k] /= 256;
+		}
+
+		for (xo = bbox2[0]; xo <= bbox2[2]; xo++) {
+			for (yo = bbox2[1]; yo <= bbox2[3]; yo++) {
+				unsigned jx = tx * span + xo;
+				unsigned jy = ty * span + yo;
+
+				// j is the shard that the child tile's data is being written to.
+				//
+				// Be careful: We can't jump more zoom levels than MAX_ZOOM_INCREMENT
+				// because that could break the constraint that each of the children
+				// of the current tile must have its own shard, because the data for
+				// the child tile must be contiguous within the shard.
+				//
+				// But it's OK to spread children across all the shards, not just
+				// the four that would normally result from splitting one tile,
+				// because it will go through all the shards when it does the
+				// next zoom.
+
+				int j = ((jx & ((1 << MAX_ZOOM_INCREMENT) - 1)) << MAX_ZOOM_INCREMENT) |
+					((jy & ((1 << MAX_ZOOM_INCREMENT) - 1)));
+
+				{
+					if (!within[j]) {
+						serialize_int(geomfile[j], nextzoom, &geompos[j], fname);
+						serialize_uint(geomfile[j], tx * span + xo, &geompos[j], fname);
+						serialize_uint(geomfile[j], ty * span + yo, &geompos[j], fname);
+						within[j] = 1;
+					}
+
+					// Offset from tile coordinates back to world coordinates
+					unsigned sx = 0, sy = 0;
+					if (z != 0) {
+						sx = tx << (32 - z);
+						sy = ty << (32 - z);
+					}
+
+					// printf("type %d, meta %lld\n", t, metastart);
+					serialize_byte(geomfile[j], t, &geompos[j], fname);
+					serialize_byte(geomfile[j], layer, &geompos[j], fname);
+					serialize_long_long(geomfile[j], metastart, &geompos[j], fname);
+					long long wx = initial_x, wy = initial_y;
+
+					for (unsigned u = 0; u < geom.size(); u++) {
+						serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname);
+
+						if (geom[u].op != VT_CLOSEPATH) {
+							serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname);
+							serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname);
+							wx = geom[u].x + sx;
+							wy = geom[u].y + sy;
+						}
+					}
+
+					serialize_byte(geomfile[j], VT_END, &geompos[j], fname);
+					serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname);
+				}
+			}
+		}
+	}
+}
+
+long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) {
 	int line_detail;
 	static bool evaluated = false;
 	double oprogress = 0;
@@ -366,6 +455,15 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 
 	char *og = *geoms;
 
+	int nextzoom = z + 1;
+	if (nextzoom < file_minzoom) {
+		if (z + MAX_ZOOM_INCREMENT > file_minzoom) {
+			nextzoom = file_minzoom;
+		} else {
+			nextzoom = z + MAX_ZOOM_INCREMENT;
+		}
+	}
+
 	for (line_detail = detail; line_detail >= min_detail || line_detail == detail; line_detail--) {
 		GOOGLE_PROTOBUF_VERIFY_VERSION;
 
@@ -404,8 +502,8 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 			features.push_back(std::vector<coalesce>());
 		}
 
-		int within[4] = {0};
+		int within[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0};
-		long long geompos[4] = {0};
+		long long geompos[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0};
 
 		*geoms = og;
 
@@ -453,7 +551,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 					geom = clip_point(geom, z, line_detail, buffer);
 				}
 
-				geom = remove_noop(geom, t);
+				geom = remove_noop(geom, t, 0);
 			}
 
 			if (geom.size() > 0) {
@@ -461,68 +559,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 			}
 
 			if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */
-				if (geom.size() > 0 && z + 1 <= file_maxzoom) {
+				rewrite(geom, z, nextzoom, file_maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom);
-					int j;
-					for (j = 0; j < 4; j++) {
-						int xo = j & 1;
-						int yo = (j >> 1) & 1;
-
-						long long bbox2[4];
-						int k;
-						for (k = 0; k < 4; k++) {
-							bbox2[k] = bbox[k];
-						}
-						if (z != 0) {
-							// Offset back to world-relative
-							bbox2[0] += tx << (32 - z);
-							bbox2[1] += ty << (32 - z);
-							bbox2[2] += tx << (32 - z);
-							bbox2[3] += ty << (32 - z);
-						}
-						// Offset to child tile-relative
-						bbox2[0] -= (tx * 2 + xo) << (32 - (z + 1));
-						bbox2[1] -= (ty * 2 + yo) << (32 - (z + 1));
-						bbox2[2] -= (tx * 2 + xo) << (32 - (z + 1));
-						bbox2[3] -= (ty * 2 + yo) << (32 - (z + 1));
-
-						int quick2 = quick_check(bbox2, z + 1, line_detail, buffer);
-						if (quick2 != 0) {
-							if (!within[j]) {
-								serialize_int(geomfile[j], z + 1, &geompos[j], fname);
-								serialize_uint(geomfile[j], tx * 2 + xo, &geompos[j], fname);
-								serialize_uint(geomfile[j], ty * 2 + yo, &geompos[j], fname);
-								within[j] = 1;
-							}
-
-							// Offset from tile coordinates back to world coordinates
-							unsigned sx = 0, sy = 0;
-							if (z != 0) {
-								sx = tx << (32 - z);
-								sy = ty << (32 - z);
-							}
-
-							// printf("type %d, meta %lld\n", t, metastart);
-							serialize_byte(geomfile[j], t, &geompos[j], fname);
-							serialize_byte(geomfile[j], layer, &geompos[j], fname);
-							serialize_long_long(geomfile[j], metastart, &geompos[j], fname);
-							long long wx = initial_x, wy = initial_y;
-
-							for (unsigned u = 0; u < geom.size(); u++) {
-								serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname);
-
-								if (geom[u].op != VT_CLOSEPATH) {
-									serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname);
-									serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname);
-									wx = geom[u].x + sx;
-									wy = geom[u].y + sy;
-								}
-							}
-
-							serialize_byte(geomfile[j], VT_END, &geompos[j], fname);
-							serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname);
-						}
-					}
-				}
 			}
 
 			if (z < file_minzoom) {
@@ -588,6 +625,10 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 
 			if ((t == VT_LINE || t == VT_POLYGON) && !prevent['s' & 0xFF]) {
 				if (!reduced) {
+					if (t == VT_LINE) {
+						geom = remove_noop(geom, t, 32 - z - line_detail);
+					}
+
 					geom = simplify_lines(geom, z, line_detail);
 				}
 			}
@@ -632,7 +673,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 		}
 
 		int j;
-		for (j = 0; j < 4; j++) {
+		for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
 			if (within[j]) {
 				serialize_byte(geomfile[j], -2, &geompos[j], fname);
 				within[j] = 0;
@@ -669,7 +710,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 
 			for (x = 0; x < features[j].size(); x++) {
 				if (features[j][x].coalesced && features[j][x].type == VT_LINE) {
-					features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type);
+					features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type, 0);
 					features[j][x].geom = simplify_lines(features[j][x].geom, 32, 0);
 				}
 			}
@@ -737,15 +778,15 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
 	return -1;
 }
 
-int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) {
+int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) {
 	int i;
 	for (i = 0; i <= maxzoom; i++) {
 		long long most = 0;
 
-		FILE *sub[4];
+		FILE *sub[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)];
-		int subfd[4];
+		int subfd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)];
 		int j;
-		for (j = 0; j < 4; j++) {
+		for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
 			char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1];
 			sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j);
 			subfd[j] = mkstemp(geomname);
@@ -764,11 +805,11 @@ int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stri
 
 		long long todo = 0;
 		long long along = 0;
-		for (j = 0; j < 4; j++) {
+		for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
 			todo += geom_size[j];
 		}
 
-		for (j = 0; j < 4; j++) {
+		for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
 			if (geomfd[j] < 0) {
 				// only one source file for zoom level 0
 				continue;
@@ -817,7 +858,7 @@ int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stri
 			along += geom_size[j];
 		}
 
-		for (j = 0; j < 4; j++) {
+		for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) {
 			close(geomfd[j]);
 			fclose(sub[j]);
 
@@ -835,4 +876,3 @@ int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stri
 	fprintf(stderr, "\n");
 	return maxzoom;
 }
-

tile.h

@@ -25,9 +25,11 @@ void deserialize_uint(char **f, unsigned *n);
 void deserialize_byte(char **f, signed char *n);
 struct pool_val *deserialize_string(char **f, struct pool *p, int type);
 
-long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent);
+long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent);
 
-int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail);
+int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail);
 
 extern unsigned initial_x, initial_y;
 extern int geometry_scale;
+
+#define MAX_ZOOM_INCREMENT 3