From efe3c62bb184b513f36e7c7c4fb9b762892bf9e2 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 29 Jun 2015 15:50:46 -0700 Subject: [PATCH 01/14] Start breaking the assumption that the next zoom is exactly 1 deeper --- geojson.c | 6 +++--- tile.cc | 4 ++-- tile.h | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/geojson.c b/geojson.c index 467b5112..26a01d06 100644 --- a/geojson.c +++ b/geojson.c @@ -954,14 +954,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max exit(EXIT_FAILURE); } - int fd[4]; - off_t size[4]; + int fd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; + off_t size[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; fd[0] = geomfd; size[0] = geomst.st_size; int j; - for (j = 1; j < 4; j++) { + for (j = 1; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { fd[j] = -1; size[j] = 0; } diff --git a/tile.cc b/tile.cc index d24a5b85..fbdfdd40 100644 --- a/tile.cc +++ b/tile.cc @@ -358,7 +358,7 @@ void evaluate(std::vector &features, char *metabase, struct pool *file } #endif -long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) { +long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) { int line_detail; static bool evaluated = false; double oprogress = 0; @@ -737,7 +737,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f return -1; } -int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) { int i; for (i = 0; i <= maxzoom; i++) { long long most = 0; diff --git a/tile.h b/tile.h index 96641e6d..93cc604a 100644 --- a/tile.h +++ b/tile.h @@ -25,9 +25,11 @@ void deserialize_uint(char **f, unsigned *n); void deserialize_byte(char **f, signed char *n); struct pool_val *deserialize_string(char **f, struct pool *p, int type); -long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent); +long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent); -int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail); extern unsigned initial_x, initial_y; extern int geometry_scale; + +#define MAX_ZOOM_INCREMENT 4 From 7724e2c3293a0e7791e20e4e3fd40b594e701410 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 29 Jun 2015 16:42:26 -0700 Subject: [PATCH 02/14] Skip over some intermediate zooms below minzoom --- tile.cc | 132 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 72 insertions(+), 60 deletions(-) diff --git a/tile.cc b/tile.cc index fbdfdd40..884ffe08 100644 --- a/tile.cc +++ b/tile.cc @@ -366,6 +366,15 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f char *og = *geoms; + int nextzoom = z + 1; + if (nextzoom < file_minzoom) { + if (z + MAX_ZOOM_INCREMENT > file_minzoom) { + nextzoom = file_minzoom; + } else { + nextzoom = z + MAX_ZOOM_INCREMENT; + } + } + for (line_detail = detail; line_detail >= min_detail || line_detail == detail; line_detail--) { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -404,8 +413,8 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f features.push_back(std::vector()); } - int within[4] = {0}; - long long geompos[4] = {0}; + int within[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0}; + long long geompos[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0}; *geoms = og; @@ -461,65 +470,68 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */ - if (geom.size() > 0 && z + 1 <= file_maxzoom) { - int j; - for (j = 0; j < 4; j++) { - int xo = j & 1; - int yo = (j >> 1) & 1; + if (geom.size() > 0 && nextzoom <= file_maxzoom) { + int xo, yo; + int span = 1 << (nextzoom - z); - long long bbox2[4]; - int k; - for (k = 0; k < 4; k++) { - bbox2[k] = bbox[k]; - } - if (z != 0) { - // Offset back to world-relative - bbox2[0] += tx << (32 - z); - bbox2[1] += ty << (32 - z); - bbox2[2] += tx << (32 - z); - bbox2[3] += ty << (32 - z); - } - // Offset to child tile-relative - bbox2[0] -= (tx * 2 + xo) << (32 - (z + 1)); - bbox2[1] -= (ty * 2 + yo) << (32 - (z + 1)); - bbox2[2] -= (tx * 2 + xo) << (32 - (z + 1)); - bbox2[3] -= (ty * 2 + yo) << (32 - (z + 1)); + for (xo = 0; xo < span; xo++) { + for (yo = 0; yo < span; yo++) { + int j = xo * span + yo; - int quick2 = quick_check(bbox2, z + 1, line_detail, buffer); - if (quick2 != 0) { - if (!within[j]) { - serialize_int(geomfile[j], z + 1, &geompos[j], fname); - serialize_uint(geomfile[j], tx * 2 + xo, &geompos[j], fname); - serialize_uint(geomfile[j], ty * 2 + yo, &geompos[j], fname); - within[j] = 1; + long long bbox2[span * span]; + int k; + for (k = 0; k < span * span; k++) { + bbox2[k] = bbox[k]; } - - // Offset from tile coordinates back to world coordinates - unsigned sx = 0, sy = 0; if (z != 0) { - sx = tx << (32 - z); - sy = ty << (32 - z); + // Offset back to world-relative + bbox2[0] += tx << (32 - z); + bbox2[1] += ty << (32 - z); + bbox2[2] += tx << (32 - z); + bbox2[3] += ty << (32 - z); } + // Offset to child tile-relative + bbox2[0] -= (tx * span + xo) << (32 - nextzoom); + bbox2[1] -= (ty * span + yo) << (32 - nextzoom); + bbox2[2] -= (tx * span + xo) << (32 - nextzoom); + bbox2[3] -= (ty * span + yo) << (32 - nextzoom); - // printf("type %d, meta %lld\n", t, metastart); - serialize_byte(geomfile[j], t, &geompos[j], fname); - serialize_byte(geomfile[j], layer, &geompos[j], fname); - serialize_long_long(geomfile[j], metastart, &geompos[j], fname); - long long wx = initial_x, wy = initial_y; - - for (unsigned u = 0; u < geom.size(); u++) { - serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname); - - if (geom[u].op != VT_CLOSEPATH) { - serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname); - serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname); - wx = geom[u].x + sx; - wy = geom[u].y + sy; + int quick2 = quick_check(bbox2, nextzoom, line_detail, buffer); + if (quick2 != 0) { + if (!within[j]) { + serialize_int(geomfile[j], nextzoom, &geompos[j], fname); + serialize_uint(geomfile[j], tx * span + xo, &geompos[j], fname); + serialize_uint(geomfile[j], ty * span + yo, &geompos[j], fname); + within[j] = 1; } - } - serialize_byte(geomfile[j], VT_END, &geompos[j], fname); - serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname); + // Offset from tile coordinates back to world coordinates + unsigned sx = 0, sy = 0; + if (z != 0) { + sx = tx << (32 - z); + sy = ty << (32 - z); + } + + // printf("type %d, meta %lld\n", t, metastart); + serialize_byte(geomfile[j], t, &geompos[j], fname); + serialize_byte(geomfile[j], layer, &geompos[j], fname); + serialize_long_long(geomfile[j], metastart, &geompos[j], fname); + long long wx = initial_x, wy = initial_y; + + for (unsigned u = 0; u < geom.size(); u++) { + serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname); + + if (geom[u].op != VT_CLOSEPATH) { + serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname); + serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname); + wx = geom[u].x + sx; + wy = geom[u].y + sy; + } + } + + serialize_byte(geomfile[j], VT_END, &geompos[j], fname); + serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname); + } } } } @@ -632,7 +644,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } int j; - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { if (within[j]) { serialize_byte(geomfile[j], -2, &geompos[j], fname); within[j] = 0; @@ -742,10 +754,10 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo for (i = 0; i <= maxzoom; i++) { long long most = 0; - FILE *sub[4]; - int subfd[4]; + FILE *sub[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; + int subfd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; int j; - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j); subfd[j] = mkstemp(geomname); @@ -764,11 +776,11 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo long long todo = 0; long long along = 0; - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { todo += geom_size[j]; } - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { if (geomfd[j] < 0) { // only one source file for zoom level 0 continue; @@ -817,7 +829,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo along += geom_size[j]; } - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { close(geomfd[j]); fclose(sub[j]); From 4bb88e228a6283f32f4fa929b6c38278d59d7261 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 29 Jun 2015 17:05:33 -0700 Subject: [PATCH 03/14] Fix buffer overflow. This is a bounding box, not sub-tiles. --- tile.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tile.cc b/tile.cc index 884ffe08..b1745689 100644 --- a/tile.cc +++ b/tile.cc @@ -478,9 +478,9 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f for (yo = 0; yo < span; yo++) { int j = xo * span + yo; - long long bbox2[span * span]; + long long bbox2[4]; int k; - for (k = 0; k < span * span; k++) { + for (k = 0; k < 4; k++) { bbox2[k] = bbox[k]; } if (z != 0) { From 530852ae000e526a7098dc8f44a16b0af27dcff4 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 29 Jun 2015 17:18:48 -0700 Subject: [PATCH 04/14] Maximum zoom increment of 3 seems to give the best performance --- tile.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tile.h b/tile.h index 93cc604a..42f96071 100644 --- a/tile.h +++ b/tile.h @@ -32,4 +32,4 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo extern unsigned initial_x, initial_y; extern int geometry_scale; -#define MAX_ZOOM_INCREMENT 4 +#define MAX_ZOOM_INCREMENT 3 From 0d3192b863e7c4f049e39bec79ad79a892b920d6 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 30 Jun 2015 16:36:26 -0700 Subject: [PATCH 05/14] Break rewriting the geometry for the next zoom out into its own function --- tile.cc | 134 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 69 insertions(+), 65 deletions(-) diff --git a/tile.cc b/tile.cc index b1745689..fab8b4c7 100644 --- a/tile.cc +++ b/tile.cc @@ -358,6 +358,74 @@ void evaluate(std::vector &features, char *metabase, struct pool *file } #endif +void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, signed char layer, long long metastart, signed char feature_minzoom) { + if (geom.size() > 0 && nextzoom <= file_maxzoom) { + int xo, yo; + int span = 1 << (nextzoom - z); + + for (xo = 0; xo < span; xo++) { + for (yo = 0; yo < span; yo++) { + int j = xo * span + yo; + + long long bbox2[4]; + int k; + for (k = 0; k < 4; k++) { + bbox2[k] = bbox[k]; + } + if (z != 0) { + // Offset back to world-relative + bbox2[0] += tx << (32 - z); + bbox2[1] += ty << (32 - z); + bbox2[2] += tx << (32 - z); + bbox2[3] += ty << (32 - z); + } + // Offset to child tile-relative + bbox2[0] -= (tx * span + xo) << (32 - nextzoom); + bbox2[1] -= (ty * span + yo) << (32 - nextzoom); + bbox2[2] -= (tx * span + xo) << (32 - nextzoom); + bbox2[3] -= (ty * span + yo) << (32 - nextzoom); + + int quick2 = quick_check(bbox2, nextzoom, line_detail, buffer); + if (quick2 != 0) { + if (!within[j]) { + serialize_int(geomfile[j], nextzoom, &geompos[j], fname); + serialize_uint(geomfile[j], tx * span + xo, &geompos[j], fname); + serialize_uint(geomfile[j], ty * span + yo, &geompos[j], fname); + within[j] = 1; + } + + // Offset from tile coordinates back to world coordinates + unsigned sx = 0, sy = 0; + if (z != 0) { + sx = tx << (32 - z); + sy = ty << (32 - z); + } + + // printf("type %d, meta %lld\n", t, metastart); + serialize_byte(geomfile[j], t, &geompos[j], fname); + serialize_byte(geomfile[j], layer, &geompos[j], fname); + serialize_long_long(geomfile[j], metastart, &geompos[j], fname); + long long wx = initial_x, wy = initial_y; + + for (unsigned u = 0; u < geom.size(); u++) { + serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname); + + if (geom[u].op != VT_CLOSEPATH) { + serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname); + serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname); + wx = geom[u].x + sx; + wy = geom[u].y + sy; + } + } + + serialize_byte(geomfile[j], VT_END, &geompos[j], fname); + serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname); + } + } + } + } +} + long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) { int line_detail; static bool evaluated = false; @@ -470,71 +538,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */ - if (geom.size() > 0 && nextzoom <= file_maxzoom) { - int xo, yo; - int span = 1 << (nextzoom - z); - - for (xo = 0; xo < span; xo++) { - for (yo = 0; yo < span; yo++) { - int j = xo * span + yo; - - long long bbox2[4]; - int k; - for (k = 0; k < 4; k++) { - bbox2[k] = bbox[k]; - } - if (z != 0) { - // Offset back to world-relative - bbox2[0] += tx << (32 - z); - bbox2[1] += ty << (32 - z); - bbox2[2] += tx << (32 - z); - bbox2[3] += ty << (32 - z); - } - // Offset to child tile-relative - bbox2[0] -= (tx * span + xo) << (32 - nextzoom); - bbox2[1] -= (ty * span + yo) << (32 - nextzoom); - bbox2[2] -= (tx * span + xo) << (32 - nextzoom); - bbox2[3] -= (ty * span + yo) << (32 - nextzoom); - - int quick2 = quick_check(bbox2, nextzoom, line_detail, buffer); - if (quick2 != 0) { - if (!within[j]) { - serialize_int(geomfile[j], nextzoom, &geompos[j], fname); - serialize_uint(geomfile[j], tx * span + xo, &geompos[j], fname); - serialize_uint(geomfile[j], ty * span + yo, &geompos[j], fname); - within[j] = 1; - } - - // Offset from tile coordinates back to world coordinates - unsigned sx = 0, sy = 0; - if (z != 0) { - sx = tx << (32 - z); - sy = ty << (32 - z); - } - - // printf("type %d, meta %lld\n", t, metastart); - serialize_byte(geomfile[j], t, &geompos[j], fname); - serialize_byte(geomfile[j], layer, &geompos[j], fname); - serialize_long_long(geomfile[j], metastart, &geompos[j], fname); - long long wx = initial_x, wy = initial_y; - - for (unsigned u = 0; u < geom.size(); u++) { - serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname); - - if (geom[u].op != VT_CLOSEPATH) { - serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname); - serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname); - wx = geom[u].x + sx; - wy = geom[u].y + sy; - } - } - - serialize_byte(geomfile[j], VT_END, &geompos[j], fname); - serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname); - } - } - } - } + rewrite(geom, z, nextzoom, file_maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom); } if (z < file_minzoom) { From 767a5818741f84ebeb4a422870881c769e751c0b Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 30 Jun 2015 16:47:25 -0700 Subject: [PATCH 06/14] Inline the I/O in the long long serialization loop --- geojson.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/geojson.c b/geojson.c index 26a01d06..4091e37f 100644 --- a/geojson.c +++ b/geojson.c @@ -75,11 +75,17 @@ void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fn unsigned char b = zigzag & 0x7F; if ((zigzag >> 7) != 0) { b |= 0x80; - fwrite_check(&b, sizeof(unsigned char), 1, out, fname); + if (putc(b, out) == EOF) { + fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); + exit(EXIT_FAILURE); + } *fpos += 1; zigzag >>= 7; } else { - fwrite_check(&b, sizeof(unsigned char), 1, out, fname); + if (putc(b, out) == EOF) { + fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); + exit(EXIT_FAILURE); + } *fpos += 1; break; } From b2fdcba6b04eb4356eab855c1d2cb55d68cf4f93 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 30 Jun 2015 17:21:48 -0700 Subject: [PATCH 07/14] Don't shrink string buffers before returning them as JSON objects. It takes a little bit of time, and it's already not shrinking them for number objects. --- jsonpull.c | 1 - 1 file changed, 1 deletion(-) diff --git a/jsonpull.c b/jsonpull.c index 5c5928d1..fc85b92d 100644 --- a/jsonpull.c +++ b/jsonpull.c @@ -499,7 +499,6 @@ again: json_object *s = add_object(j, JSON_STRING); if (s != NULL) { - val.buf = realloc(val.buf, val.n + 1); s->string = val.buf; s->length = val.n; } else { From af13a95dc11828656a8a36158a9b6a8fb5e2783e Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 1 Jul 2015 10:12:23 -0700 Subject: [PATCH 08/14] Shard the child tiles as widely as possible even if not skipping levels to help future parallelization --- tile.cc | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tile.cc b/tile.cc index fab8b4c7..1c36762c 100644 --- a/tile.cc +++ b/tile.cc @@ -365,7 +365,23 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb for (xo = 0; xo < span; xo++) { for (yo = 0; yo < span; yo++) { - int j = xo * span + yo; + unsigned jx = tx * span + xo; + unsigned jy = ty * span + yo; + + // j is the shard that the child tile's data is being written to. + // + // Be careful: We can't jump more zoom levels than MAX_ZOOM_INCREMENT + // because that could break the constraint that each of the children + // of the current tile must have its own shard, because the data for + // the child tile must be contiguous within the shard. + // + // But it's OK to spread children across all the shards, not just + // the four that would normally result from splitting one tile, + // because it will go through all the shards when it does the + // next zoom. + + int j = ((jx & ((1 << MAX_ZOOM_INCREMENT) - 1)) << MAX_ZOOM_INCREMENT) | + ((jy & ((1 << MAX_ZOOM_INCREMENT) - 1))); long long bbox2[4]; int k; From 18cdcb07325f77c6ab3cc780b7c3f72466c143f0 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 1 Jul 2015 12:16:50 -0700 Subject: [PATCH 09/14] Drop linetos smaller than tile resolution before doing normal simplification Since simplification is rather expensive --- geometry.cc | 10 +++++----- geometry.hh | 2 +- tile.cc | 10 +++++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/geometry.cc b/geometry.cc index 281a41ca..b0f2e862 100644 --- a/geometry.cc +++ b/geometry.cc @@ -84,7 +84,7 @@ void to_tile_scale(drawvec &geom, int z, int detail) { } } -drawvec remove_noop(drawvec geom, int type) { +drawvec remove_noop(drawvec geom, int type, int shift) { // first pass: remove empty linetos long long x = 0, y = 0; @@ -92,7 +92,7 @@ drawvec remove_noop(drawvec geom, int type) { unsigned i; for (i = 0; i < geom.size(); i++) { - if (geom[i].op == VT_LINETO && geom[i].x == x && geom[i].y == y) { + if (geom[i].op == VT_LINETO && (geom[i].x >> shift) == x && (geom[i].y >> shift) == y) { continue; } @@ -100,8 +100,8 @@ drawvec remove_noop(drawvec geom, int type) { out.push_back(geom[i]); } else { /* moveto or lineto */ out.push_back(geom[i]); - x = geom[i].x; - y = geom[i].y; + x = geom[i].x >> shift; + y = geom[i].y >> shift; } } @@ -137,7 +137,7 @@ drawvec remove_noop(drawvec geom, int type) { for (i = 0; i < geom.size(); i++) { if (geom[i].op == VT_MOVETO) { - if (i > 0 && geom[i - 1].op == VT_LINETO && geom[i - 1].x == geom[i].x && geom[i - 1].y == geom[i].y) { + if (i > 0 && geom[i - 1].op == VT_LINETO && (geom[i - 1].x >> shift) == (geom[i].x >> shift) && (geom[i - 1].y >> shift) == (geom[i].y >> shift)) { continue; } } diff --git a/geometry.hh b/geometry.hh index b4cb3e9d..8848e405 100644 --- a/geometry.hh +++ b/geometry.hh @@ -18,7 +18,7 @@ typedef std::vector drawvec; drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox); void to_tile_scale(drawvec &geom, int z, int detail); -drawvec remove_noop(drawvec geom, int type); +drawvec remove_noop(drawvec geom, int type, int shift); drawvec clip_point(drawvec &geom, int z, int detail, long long buffer); drawvec clip_poly(drawvec &geom, int z, int detail, int buffer); drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double *accum_area); diff --git a/tile.cc b/tile.cc index 1c36762c..8f1b42fa 100644 --- a/tile.cc +++ b/tile.cc @@ -236,7 +236,7 @@ mapnik::vector::tile create_tile(char **layernames, int line_detail, std::vector unsigned x; for (x = 0; x < features[i].size(); x++) { if (features[i][x].type == VT_LINE || features[i][x].type == VT_POLYGON) { - features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type); + features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type, 0); } mapnik::vector::tile_feature *feature = layer->add_features(); @@ -546,7 +546,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f geom = clip_point(geom, z, line_detail, buffer); } - geom = remove_noop(geom, t); + geom = remove_noop(geom, t, 0); } if (geom.size() > 0) { @@ -620,6 +620,10 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f if ((t == VT_LINE || t == VT_POLYGON) && !prevent['s' & 0xFF]) { if (!reduced) { + if (t == VT_LINE) { + geom = remove_noop(geom, t, 32 - z - line_detail); + } + geom = simplify_lines(geom, z, line_detail); } } @@ -701,7 +705,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f for (x = 0; x < features[j].size(); x++) { if (features[j][x].coalesced && features[j][x].type == VT_LINE) { - features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type); + features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type, 0); features[j][x].geom = simplify_lines(features[j][x].geom, 32, 0); } } From 1b728043580da6f22cd101eb17445d5517f6d2c4 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 1 Jul 2015 15:06:12 -0700 Subject: [PATCH 10/14] Precalculate which child tiles a feature can overlap instead of looping over them all --- tile.cc | 49 +++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/tile.cc b/tile.cc index 8f1b42fa..2c0ab3fb 100644 --- a/tile.cc +++ b/tile.cc @@ -363,8 +363,32 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb int xo, yo; int span = 1 << (nextzoom - z); - for (xo = 0; xo < span; xo++) { - for (yo = 0; yo < span; yo++) { + // Get the feature bounding box in pixel (256) coordinates at the child zoom + // in order to calculate which sub-tiles it can touch including the buffer. + long long bbox2[4]; + int k; + for (k = 0; k < 4; k++) { + // Division instead of right-shift because coordinates can be negative + bbox2[k] = bbox[k] / (1 << (32 - nextzoom - 8)); + } + bbox2[0] -= buffer; + bbox2[1] -= buffer; + bbox2[2] += buffer; + bbox2[3] += buffer; + + for (k = 0; k < 4; k++) { + if (bbox2[k] < 0) { + bbox2[k] = 0; + } + if (bbox2[k] >= 256 * span) { + bbox2[k] = 256 * (span - 1); + } + + bbox2[k] /= 256; + } + + for (xo = bbox2[0]; xo <= bbox2[2]; xo++) { + for (yo = bbox2[1]; yo <= bbox2[3]; yo++) { unsigned jx = tx * span + xo; unsigned jy = ty * span + yo; @@ -383,26 +407,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb int j = ((jx & ((1 << MAX_ZOOM_INCREMENT) - 1)) << MAX_ZOOM_INCREMENT) | ((jy & ((1 << MAX_ZOOM_INCREMENT) - 1))); - long long bbox2[4]; - int k; - for (k = 0; k < 4; k++) { - bbox2[k] = bbox[k]; - } - if (z != 0) { - // Offset back to world-relative - bbox2[0] += tx << (32 - z); - bbox2[1] += ty << (32 - z); - bbox2[2] += tx << (32 - z); - bbox2[3] += ty << (32 - z); - } - // Offset to child tile-relative - bbox2[0] -= (tx * span + xo) << (32 - nextzoom); - bbox2[1] -= (ty * span + yo) << (32 - nextzoom); - bbox2[2] -= (tx * span + xo) << (32 - nextzoom); - bbox2[3] -= (ty * span + yo) << (32 - nextzoom); - - int quick2 = quick_check(bbox2, nextzoom, line_detail, buffer); - if (quick2 != 0) { + { if (!within[j]) { serialize_int(geomfile[j], nextzoom, &geompos[j], fname); serialize_uint(geomfile[j], tx * span + xo, &geompos[j], fname); From 65f3737325490e35501df9d64ca8a4728edc776d Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 2 Jul 2015 17:03:07 -0700 Subject: [PATCH 11/14] Use a hash table of binary trees for string pooling, not just a binary tree --- pool.c | 29 ++++++++++++++++++++--------- pool.h | 2 +- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pool.c b/pool.c index a36140de..e874d23e 100644 --- a/pool.c +++ b/pool.c @@ -2,11 +2,23 @@ #include #include "pool.h" -static struct pool_val *pool1(struct pool *p, char *s, int type, int (*compare)(const char *, const char *)) { - struct pool_val **v = &(p->vals); +#define POOL_WIDTH 256 + +static int hash(char *s) { + int h = 0; + for (; *s; s++) { + h = h * 37 + *s; + } + h = h & 0xFF; + return h; +} + +struct pool_val *pool(struct pool *p, char *s, int type) { + int h = hash(s); + struct pool_val **v = &(p->vals[h]); while (*v != NULL) { - int cmp = compare(s, (*v)->s); + int cmp = strcmp(s, (*v)->s); if (cmp == 0) { cmp = type - (*v)->type; @@ -41,7 +53,8 @@ static struct pool_val *pool1(struct pool *p, char *s, int type, int (*compare)( } int is_pooled(struct pool *p, char *s, int type) { - struct pool_val **v = &(p->vals); + int h = hash(s); + struct pool_val **v = &(p->vals[h]); while (*v != NULL) { int cmp = strcmp(s, (*v)->s); @@ -62,10 +75,6 @@ int is_pooled(struct pool *p, char *s, int type) { return 0; } -struct pool_val *pool(struct pool *p, char *s, int type) { - return pool1(p, s, type, strcmp); -} - void pool_free1(struct pool *p, void (*func)(void *)) { while (p->head != NULL) { if (func != NULL) { @@ -79,6 +88,8 @@ void pool_free1(struct pool *p, void (*func)(void *)) { p->head = NULL; p->tail = NULL; + + free(p->vals); p->vals = NULL; } @@ -92,7 +103,7 @@ void pool_free_strings(struct pool *p) { void pool_init(struct pool *p, int n) { p->n = n; - p->vals = NULL; + p->vals = calloc(POOL_WIDTH, sizeof(struct pool_val *)); p->head = NULL; p->tail = NULL; } diff --git a/pool.h b/pool.h index d995e5b3..147e8c81 100644 --- a/pool.h +++ b/pool.h @@ -10,7 +10,7 @@ struct pool_val { }; struct pool { - struct pool_val *vals; + struct pool_val **vals; struct pool_val *head; struct pool_val *tail; From c19c913bf938d14906087e60fcbb35196f8a939c Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 8 Jul 2015 11:52:22 -0700 Subject: [PATCH 12/14] Fix confusing error messages from sloppy copying and pasting of code --- mbtiles.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mbtiles.c b/mbtiles.c index 4735aeac..406c4b79 100644 --- a/mbtiles.c +++ b/mbtiles.c @@ -228,11 +228,11 @@ void mbtiles_close(sqlite3 *outdb, char **argv) { char *err; if (sqlite3_exec(outdb, "ANALYZE;", NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "%s: index metadata: %s\n", argv[0], err); + fprintf(stderr, "%s: ANALYZE failed: %s\n", argv[0], err); exit(EXIT_FAILURE); } if (sqlite3_exec(outdb, "VACUUM;", NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "%s: index tiles: %s\n", argv[0], err); + fprintf(stderr, "%s: VACUUM failed: %s\n", argv[0], err); exit(EXIT_FAILURE); } if (sqlite3_close(outdb) != SQLITE_OK) { From 6fd72d451828347a75dc65ce56b5e43e81071ae9 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 8 Jul 2015 12:25:09 -0700 Subject: [PATCH 13/14] Remove vacuum step, which doesn't seem to make .mbtiles smaller in practice yet takes time and temporary disk space to perform --- mbtiles.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mbtiles.c b/mbtiles.c index 406c4b79..008170d1 100644 --- a/mbtiles.c +++ b/mbtiles.c @@ -231,10 +231,6 @@ void mbtiles_close(sqlite3 *outdb, char **argv) { fprintf(stderr, "%s: ANALYZE failed: %s\n", argv[0], err); exit(EXIT_FAILURE); } - if (sqlite3_exec(outdb, "VACUUM;", NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "%s: VACUUM failed: %s\n", argv[0], err); - exit(EXIT_FAILURE); - } if (sqlite3_close(outdb) != SQLITE_OK) { fprintf(stderr, "%s: could not close database: %s\n", argv[0], sqlite3_errmsg(outdb)); exit(EXIT_FAILURE); From e7b4443838b257877c00e168c23ce3e700dce225 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Wed, 8 Jul 2015 14:42:12 -0700 Subject: [PATCH 14/14] Enforce code style --- tile.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tile.cc b/tile.cc index 2c0ab3fb..ec277000 100644 --- a/tile.cc +++ b/tile.cc @@ -405,7 +405,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb // next zoom. int j = ((jx & ((1 << MAX_ZOOM_INCREMENT) - 1)) << MAX_ZOOM_INCREMENT) | - ((jy & ((1 << MAX_ZOOM_INCREMENT) - 1))); + ((jy & ((1 << MAX_ZOOM_INCREMENT) - 1))); { if (!within[j]) { @@ -876,4 +876,3 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo fprintf(stderr, "\n"); return maxzoom; } -