diff --git a/geojson.c b/geojson.c index 467b5112..4091e37f 100644 --- a/geojson.c +++ b/geojson.c @@ -75,11 +75,17 @@ void serialize_long_long(FILE *out, long long n, long long *fpos, const char *fn unsigned char b = zigzag & 0x7F; if ((zigzag >> 7) != 0) { b |= 0x80; - fwrite_check(&b, sizeof(unsigned char), 1, out, fname); + if (putc(b, out) == EOF) { + fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); + exit(EXIT_FAILURE); + } *fpos += 1; zigzag >>= 7; } else { - fwrite_check(&b, sizeof(unsigned char), 1, out, fname); + if (putc(b, out) == EOF) { + fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno)); + exit(EXIT_FAILURE); + } *fpos += 1; break; } @@ -954,14 +960,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max exit(EXIT_FAILURE); } - int fd[4]; - off_t size[4]; + int fd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; + off_t size[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; fd[0] = geomfd; size[0] = geomst.st_size; int j; - for (j = 1; j < 4; j++) { + for (j = 1; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { fd[j] = -1; size[j] = 0; } diff --git a/geometry.cc b/geometry.cc index 281a41ca..b0f2e862 100644 --- a/geometry.cc +++ b/geometry.cc @@ -84,7 +84,7 @@ void to_tile_scale(drawvec &geom, int z, int detail) { } } -drawvec remove_noop(drawvec geom, int type) { +drawvec remove_noop(drawvec geom, int type, int shift) { // first pass: remove empty linetos long long x = 0, y = 0; @@ -92,7 +92,7 @@ drawvec remove_noop(drawvec geom, int type) { unsigned i; for (i = 0; i < geom.size(); i++) { - if (geom[i].op == VT_LINETO && geom[i].x == x && geom[i].y == y) { + if (geom[i].op == VT_LINETO && (geom[i].x >> shift) == x && (geom[i].y >> shift) == y) { continue; } @@ -100,8 +100,8 @@ drawvec remove_noop(drawvec geom, int type) { out.push_back(geom[i]); } else { /* moveto or lineto */ out.push_back(geom[i]); - x = geom[i].x; - y = geom[i].y; + x = geom[i].x >> shift; + y = geom[i].y >> shift; } } @@ -137,7 +137,7 @@ drawvec remove_noop(drawvec geom, int type) { for (i = 0; i < geom.size(); i++) { if (geom[i].op == VT_MOVETO) { - if (i > 0 && geom[i - 1].op == VT_LINETO && geom[i - 1].x == geom[i].x && geom[i - 1].y == geom[i].y) { + if (i > 0 && geom[i - 1].op == VT_LINETO && (geom[i - 1].x >> shift) == (geom[i].x >> shift) && (geom[i - 1].y >> shift) == (geom[i].y >> shift)) { continue; } } diff --git a/geometry.hh b/geometry.hh index b4cb3e9d..8848e405 100644 --- a/geometry.hh +++ b/geometry.hh @@ -18,7 +18,7 @@ typedef std::vector drawvec; drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, int detail, long long *bbox); void to_tile_scale(drawvec &geom, int z, int detail); -drawvec remove_noop(drawvec geom, int type); +drawvec remove_noop(drawvec geom, int type, int shift); drawvec clip_point(drawvec &geom, int z, int detail, long long buffer); drawvec clip_poly(drawvec &geom, int z, int detail, int buffer); drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double *accum_area); diff --git a/jsonpull.c b/jsonpull.c index 5c5928d1..fc85b92d 100644 --- a/jsonpull.c +++ b/jsonpull.c @@ -499,7 +499,6 @@ again: json_object *s = add_object(j, JSON_STRING); if (s != NULL) { - val.buf = realloc(val.buf, val.n + 1); s->string = val.buf; s->length = val.n; } else { diff --git a/mbtiles.c b/mbtiles.c index 4735aeac..008170d1 100644 --- a/mbtiles.c +++ b/mbtiles.c @@ -228,11 +228,7 @@ void mbtiles_close(sqlite3 *outdb, char **argv) { char *err; if (sqlite3_exec(outdb, "ANALYZE;", NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "%s: index metadata: %s\n", argv[0], err); - exit(EXIT_FAILURE); - } - if (sqlite3_exec(outdb, "VACUUM;", NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "%s: index tiles: %s\n", argv[0], err); + fprintf(stderr, "%s: ANALYZE failed: %s\n", argv[0], err); exit(EXIT_FAILURE); } if (sqlite3_close(outdb) != SQLITE_OK) { diff --git a/pool.c b/pool.c index a36140de..e874d23e 100644 --- a/pool.c +++ b/pool.c @@ -2,11 +2,23 @@ #include #include "pool.h" -static struct pool_val *pool1(struct pool *p, char *s, int type, int (*compare)(const char *, const char *)) { - struct pool_val **v = &(p->vals); +#define POOL_WIDTH 256 + +static int hash(char *s) { + int h = 0; + for (; *s; s++) { + h = h * 37 + *s; + } + h = h & 0xFF; + return h; +} + +struct pool_val *pool(struct pool *p, char *s, int type) { + int h = hash(s); + struct pool_val **v = &(p->vals[h]); while (*v != NULL) { - int cmp = compare(s, (*v)->s); + int cmp = strcmp(s, (*v)->s); if (cmp == 0) { cmp = type - (*v)->type; @@ -41,7 +53,8 @@ static struct pool_val *pool1(struct pool *p, char *s, int type, int (*compare)( } int is_pooled(struct pool *p, char *s, int type) { - struct pool_val **v = &(p->vals); + int h = hash(s); + struct pool_val **v = &(p->vals[h]); while (*v != NULL) { int cmp = strcmp(s, (*v)->s); @@ -62,10 +75,6 @@ int is_pooled(struct pool *p, char *s, int type) { return 0; } -struct pool_val *pool(struct pool *p, char *s, int type) { - return pool1(p, s, type, strcmp); -} - void pool_free1(struct pool *p, void (*func)(void *)) { while (p->head != NULL) { if (func != NULL) { @@ -79,6 +88,8 @@ void pool_free1(struct pool *p, void (*func)(void *)) { p->head = NULL; p->tail = NULL; + + free(p->vals); p->vals = NULL; } @@ -92,7 +103,7 @@ void pool_free_strings(struct pool *p) { void pool_init(struct pool *p, int n) { p->n = n; - p->vals = NULL; + p->vals = calloc(POOL_WIDTH, sizeof(struct pool_val *)); p->head = NULL; p->tail = NULL; } diff --git a/pool.h b/pool.h index d995e5b3..147e8c81 100644 --- a/pool.h +++ b/pool.h @@ -10,7 +10,7 @@ struct pool_val { }; struct pool { - struct pool_val *vals; + struct pool_val **vals; struct pool_val *head; struct pool_val *tail; diff --git a/tile.cc b/tile.cc index d24a5b85..ec277000 100644 --- a/tile.cc +++ b/tile.cc @@ -236,7 +236,7 @@ mapnik::vector::tile create_tile(char **layernames, int line_detail, std::vector unsigned x; for (x = 0; x < features[i].size(); x++) { if (features[i][x].type == VT_LINE || features[i][x].type == VT_POLYGON) { - features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type); + features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type, 0); } mapnik::vector::tile_feature *feature = layer->add_features(); @@ -358,7 +358,96 @@ void evaluate(std::vector &features, char *metabase, struct pool *file } #endif -long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) { +void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, signed char layer, long long metastart, signed char feature_minzoom) { + if (geom.size() > 0 && nextzoom <= file_maxzoom) { + int xo, yo; + int span = 1 << (nextzoom - z); + + // Get the feature bounding box in pixel (256) coordinates at the child zoom + // in order to calculate which sub-tiles it can touch including the buffer. + long long bbox2[4]; + int k; + for (k = 0; k < 4; k++) { + // Division instead of right-shift because coordinates can be negative + bbox2[k] = bbox[k] / (1 << (32 - nextzoom - 8)); + } + bbox2[0] -= buffer; + bbox2[1] -= buffer; + bbox2[2] += buffer; + bbox2[3] += buffer; + + for (k = 0; k < 4; k++) { + if (bbox2[k] < 0) { + bbox2[k] = 0; + } + if (bbox2[k] >= 256 * span) { + bbox2[k] = 256 * (span - 1); + } + + bbox2[k] /= 256; + } + + for (xo = bbox2[0]; xo <= bbox2[2]; xo++) { + for (yo = bbox2[1]; yo <= bbox2[3]; yo++) { + unsigned jx = tx * span + xo; + unsigned jy = ty * span + yo; + + // j is the shard that the child tile's data is being written to. + // + // Be careful: We can't jump more zoom levels than MAX_ZOOM_INCREMENT + // because that could break the constraint that each of the children + // of the current tile must have its own shard, because the data for + // the child tile must be contiguous within the shard. + // + // But it's OK to spread children across all the shards, not just + // the four that would normally result from splitting one tile, + // because it will go through all the shards when it does the + // next zoom. + + int j = ((jx & ((1 << MAX_ZOOM_INCREMENT) - 1)) << MAX_ZOOM_INCREMENT) | + ((jy & ((1 << MAX_ZOOM_INCREMENT) - 1))); + + { + if (!within[j]) { + serialize_int(geomfile[j], nextzoom, &geompos[j], fname); + serialize_uint(geomfile[j], tx * span + xo, &geompos[j], fname); + serialize_uint(geomfile[j], ty * span + yo, &geompos[j], fname); + within[j] = 1; + } + + // Offset from tile coordinates back to world coordinates + unsigned sx = 0, sy = 0; + if (z != 0) { + sx = tx << (32 - z); + sy = ty << (32 - z); + } + + // printf("type %d, meta %lld\n", t, metastart); + serialize_byte(geomfile[j], t, &geompos[j], fname); + serialize_byte(geomfile[j], layer, &geompos[j], fname); + serialize_long_long(geomfile[j], metastart, &geompos[j], fname); + long long wx = initial_x, wy = initial_y; + + for (unsigned u = 0; u < geom.size(); u++) { + serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname); + + if (geom[u].op != VT_CLOSEPATH) { + serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname); + serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname); + wx = geom[u].x + sx; + wy = geom[u].y + sy; + } + } + + serialize_byte(geomfile[j], VT_END, &geompos[j], fname); + serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname); + } + } + } + } +} + +long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent) { int line_detail; static bool evaluated = false; double oprogress = 0; @@ -366,6 +455,15 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f char *og = *geoms; + int nextzoom = z + 1; + if (nextzoom < file_minzoom) { + if (z + MAX_ZOOM_INCREMENT > file_minzoom) { + nextzoom = file_minzoom; + } else { + nextzoom = z + MAX_ZOOM_INCREMENT; + } + } + for (line_detail = detail; line_detail >= min_detail || line_detail == detail; line_detail--) { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -404,8 +502,8 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f features.push_back(std::vector()); } - int within[4] = {0}; - long long geompos[4] = {0}; + int within[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0}; + long long geompos[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0}; *geoms = og; @@ -453,7 +551,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f geom = clip_point(geom, z, line_detail, buffer); } - geom = remove_noop(geom, t); + geom = remove_noop(geom, t, 0); } if (geom.size() > 0) { @@ -461,68 +559,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */ - if (geom.size() > 0 && z + 1 <= file_maxzoom) { - int j; - for (j = 0; j < 4; j++) { - int xo = j & 1; - int yo = (j >> 1) & 1; - - long long bbox2[4]; - int k; - for (k = 0; k < 4; k++) { - bbox2[k] = bbox[k]; - } - if (z != 0) { - // Offset back to world-relative - bbox2[0] += tx << (32 - z); - bbox2[1] += ty << (32 - z); - bbox2[2] += tx << (32 - z); - bbox2[3] += ty << (32 - z); - } - // Offset to child tile-relative - bbox2[0] -= (tx * 2 + xo) << (32 - (z + 1)); - bbox2[1] -= (ty * 2 + yo) << (32 - (z + 1)); - bbox2[2] -= (tx * 2 + xo) << (32 - (z + 1)); - bbox2[3] -= (ty * 2 + yo) << (32 - (z + 1)); - - int quick2 = quick_check(bbox2, z + 1, line_detail, buffer); - if (quick2 != 0) { - if (!within[j]) { - serialize_int(geomfile[j], z + 1, &geompos[j], fname); - serialize_uint(geomfile[j], tx * 2 + xo, &geompos[j], fname); - serialize_uint(geomfile[j], ty * 2 + yo, &geompos[j], fname); - within[j] = 1; - } - - // Offset from tile coordinates back to world coordinates - unsigned sx = 0, sy = 0; - if (z != 0) { - sx = tx << (32 - z); - sy = ty << (32 - z); - } - - // printf("type %d, meta %lld\n", t, metastart); - serialize_byte(geomfile[j], t, &geompos[j], fname); - serialize_byte(geomfile[j], layer, &geompos[j], fname); - serialize_long_long(geomfile[j], metastart, &geompos[j], fname); - long long wx = initial_x, wy = initial_y; - - for (unsigned u = 0; u < geom.size(); u++) { - serialize_byte(geomfile[j], geom[u].op, &geompos[j], fname); - - if (geom[u].op != VT_CLOSEPATH) { - serialize_long_long(geomfile[j], ((geom[u].x + sx) >> geometry_scale) - (wx >> geometry_scale), &geompos[j], fname); - serialize_long_long(geomfile[j], ((geom[u].y + sy) >> geometry_scale) - (wy >> geometry_scale), &geompos[j], fname); - wx = geom[u].x + sx; - wy = geom[u].y + sy; - } - } - - serialize_byte(geomfile[j], VT_END, &geompos[j], fname); - serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname); - } - } - } + rewrite(geom, z, nextzoom, file_maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom); } if (z < file_minzoom) { @@ -588,6 +625,10 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f if ((t == VT_LINE || t == VT_POLYGON) && !prevent['s' & 0xFF]) { if (!reduced) { + if (t == VT_LINE) { + geom = remove_noop(geom, t, 32 - z - line_detail); + } + geom = simplify_lines(geom, z, line_detail); } } @@ -632,7 +673,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } int j; - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { if (within[j]) { serialize_byte(geomfile[j], -2, &geompos[j], fname); within[j] = 0; @@ -669,7 +710,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f for (x = 0; x < features[j].size(); x++) { if (features[j][x].coalesced && features[j][x].type == VT_LINE) { - features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type); + features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type, 0); features[j][x].geom = simplify_lines(features[j][x].geom, 32, 0); } } @@ -737,15 +778,15 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f return -1; } -int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail) { int i; for (i = 0; i <= maxzoom; i++) { long long most = 0; - FILE *sub[4]; - int subfd[4]; + FILE *sub[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; + int subfd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; int j; - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { char geomname[strlen(tmpdir) + strlen("/geom2.XXXXXXXX") + 1]; sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j); subfd[j] = mkstemp(geomname); @@ -764,11 +805,11 @@ int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stri long long todo = 0; long long along = 0; - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { todo += geom_size[j]; } - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { if (geomfd[j] < 0) { // only one source file for zoom level 0 continue; @@ -817,7 +858,7 @@ int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stri along += geom_size[j]; } - for (j = 0; j < 4; j++) { + for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { close(geomfd[j]); fclose(sub[j]); @@ -835,4 +876,3 @@ int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stri fprintf(stderr, "\n"); return maxzoom; } - diff --git a/tile.h b/tile.h index 96641e6d..42f96071 100644 --- a/tile.h +++ b/tile.h @@ -25,9 +25,11 @@ void deserialize_uint(char **f, unsigned *n); void deserialize_byte(char **f, signed char *n); struct pool_val *deserialize_string(char **f, struct pool *p, int type); -long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE *geomfile[4], int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent); +long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent); -int traverse_zooms(int geomfd[4], off_t geom_size[4], char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, int full_detail, int low_detail, int min_detail); extern unsigned initial_x, initial_y; extern int geometry_scale; + +#define MAX_ZOOM_INCREMENT 3