diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dad6e99..b212f7d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## 1.3.0 + +* Tile generation is multithreaded to take advantage of multiple CPUs +* More compact data representation reduces memory usage and improves speed +* Polygon clipping uses [Clipper](http://www.angusj.com/delphi/clipper/documentation/Docs/_Body.htm) + and makes sure interior and exterior rings are distinguished by winding order +* Individual GeoJSON features can specify their own minzoom and maxzoom +* New `tile-join` utility can add new properties from a CSV file to an existing tileset +* Feature coalescing, line-reversing, and reordering by attribute are now options, not defaults +* Output of `decode` utility is now in GeoJSON format +* Tile generation with a minzoom spends less time on unused lower zoom levels +* Bare geometries without a Feature wrapper are accepted + ## 1.2.0 * Switched to top-down rendering, yielding performance improvements diff --git a/Makefile b/Makefile index c9e0d02c..0ffc05f9 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ INCLUDES = -I/usr/local/include LIBS = -L/usr/local/lib tippecanoe: geojson.o jsonpull.o vector_tile.pb.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o - g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 + g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 -lpthread enumerate: enumerate.o gcc $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lsqlite3 diff --git a/decode.cc b/decode.cc index fb977809..e6062ab1 100644 --- a/decode.cc +++ b/decode.cc @@ -417,6 +417,5 @@ int main(int argc, char **argv) { usage(argv); } - return 0; } diff --git a/geojson.c b/geojson.c index 6696d72e..064c9f90 100644 --- a/geojson.c +++ b/geojson.c @@ -1073,14 +1073,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max exit(EXIT_FAILURE); } - int fd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; - off_t size[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; + int fd[TEMP_FILES]; + off_t size[TEMP_FILES]; fd[0] = geomfd; size[0] = geomst.st_size; int j; - for (j = 1; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { + for (j = 1; j < TEMP_FILES; j++) { fd[j] = -1; size[j] = 0; } @@ -1089,7 +1089,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", seq, (long long) geomst.st_size, (long long) metast.st_size, poolfile->off); } - int written = traverse_zooms(fd, size, meta, stringpool, file_bbox, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail); + int written = traverse_zooms(fd, size, meta, stringpool, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); diff --git a/geometry.cc b/geometry.cc index 508b3083..482142d6 100644 --- a/geometry.cc +++ b/geometry.cc @@ -339,11 +339,11 @@ drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double if (*accum_area > pixel * pixel) { // XXX use centroid; - out.push_back(draw(VT_MOVETO, geom[i].x - pixel/2, geom[i].y - pixel/2)); - out.push_back(draw(VT_LINETO, geom[i].x + pixel/2, geom[i].y - pixel/2)); - out.push_back(draw(VT_LINETO, geom[i].x + pixel/2, geom[i].y + pixel/2)); - out.push_back(draw(VT_LINETO, geom[i].x - pixel/2, geom[i].y + pixel/2)); - out.push_back(draw(VT_LINETO, geom[i].x - pixel/2, geom[i].y - pixel/2)); + out.push_back(draw(VT_MOVETO, geom[i].x - pixel / 2, geom[i].y - pixel / 2)); + out.push_back(draw(VT_LINETO, geom[i].x + pixel / 2, geom[i].y - pixel / 2)); + out.push_back(draw(VT_LINETO, geom[i].x + pixel / 2, geom[i].y + pixel / 2)); + out.push_back(draw(VT_LINETO, geom[i].x - pixel / 2, geom[i].y + pixel / 2)); + out.push_back(draw(VT_LINETO, geom[i].x - pixel / 2, geom[i].y - pixel / 2)); *accum_area -= pixel * pixel; } diff --git a/pool.c b/pool.c index 4641afc5..4c46bef5 100644 --- a/pool.c +++ b/pool.c @@ -34,26 +34,27 @@ struct pool_val *pool(struct pool *p, const char *s, int type) { } } - *v = malloc(sizeof(struct pool_val)); - if (*v == NULL) { + struct pool_val *nv = malloc(sizeof(struct pool_val)); + if (nv == NULL) { fprintf(stderr, "out of memory making string pool\n"); exit(EXIT_FAILURE); } - (*v)->left = NULL; - (*v)->right = NULL; - (*v)->next = NULL; - (*v)->s = s; - (*v)->type = type; - (*v)->n = p->n++; + nv->left = NULL; + nv->right = NULL; + nv->next = NULL; + nv->s = s; + nv->type = type; + nv->n = p->n++; if (p->tail != NULL) { - p->tail->next = *v; + p->tail->next = nv; } - p->tail = *v; + p->tail = nv; if (p->head == NULL) { - p->head = *v; + p->head = nv; } + *v = nv; return *v; } diff --git a/tile.cc b/tile.cc index 9b461a85..5e4cb44d 100644 --- a/tile.cc +++ b/tile.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include "vector_tile.pb.h" #include "geometry.hh" @@ -31,6 +32,9 @@ extern "C" { #define XSTRINGIFY(s) STRINGIFY(s) #define STRINGIFY(s) #s +pthread_mutex_t db_lock = PTHREAD_MUTEX_INITIALIZER; +pthread_mutex_t var_lock = PTHREAD_MUTEX_INITIALIZER; + // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp static inline int compress(std::string const &input, std::string &output) { z_stream deflate_s; @@ -208,26 +212,30 @@ struct pool_val *retrieve_string(char **f, struct pool *p, char *stringpool) { return ret; } -void decode_meta(char **meta, char *stringpool, struct pool *keys, struct pool *values, struct pool *file_keys, std::vector *intmeta, char *only) { +void decode_meta(char **meta, char *stringpool, struct pool *keys, struct pool *values, struct pool *file_keys, std::vector *intmeta) { int m; deserialize_int(meta, &m); int i; for (i = 0; i < m; i++) { struct pool_val *key = retrieve_string(meta, keys, stringpool); + struct pool_val *value = retrieve_string(meta, values, stringpool); - if (only != NULL && (strcmp(key->s, only) != 0)) { - // XXX if evaluate ever works again, check whether this is sufficient - (void) retrieve_string(meta, values, stringpool); - } else { - struct pool_val *value = retrieve_string(meta, values, stringpool); + intmeta->push_back(key->n); + intmeta->push_back(value->n); - intmeta->push_back(key->n); - intmeta->push_back(value->n); + if (!is_pooled(file_keys, key->s, value->type)) { + if (pthread_mutex_lock(&var_lock) != 0) { + perror("pthread_mutex_lock"); + exit(EXIT_FAILURE); + } - if (!is_pooled(file_keys, key->s, value->type)) { - // Dup to retain after munmap - pool(file_keys, strdup(key->s), value->type); + // Dup to retain after munmap + pool(file_keys, strdup(key->s), value->type); + + if (pthread_mutex_unlock(&var_lock) != 0) { + perror("pthread_mutex_unlock"); + exit(EXIT_FAILURE); } } } @@ -309,67 +317,7 @@ struct sll { } }; -#if 0 -void evaluate(std::vector &features, char *metabase, struct pool *file_keys, const char *layername, int line_detail, long long orig) { - std::vector options; - - struct pool_val *pv; - for (pv = file_keys->head; pv != NULL; pv = pv->next) { - struct pool keys, values; - pool_init(&keys, 0); - pool_init(&values, 0); - long long count = 0; - - for (unsigned i = 0; i < features.size(); i++) { - char *meta = features[i].metasrc; - - features[i].meta.resize(0); - decode_meta(&meta, &keys, &values, file_keys, &features[i].meta, pv->s); - } - - std::vector empty; - mapnik::vector::tile tile = create_tile(layername, line_detail, empty, &count, &keys, &values, 1); // XXX layer - - std::string s; - std::string compressed; - - tile.SerializeToString(&s); - compress(s, compressed); - - options.push_back(sll(pv->s, compressed.size())); - - pool_free(&values); - pool_free(&keys); - } - - std::sort(options.begin(), options.end()); - for (unsigned i = 0; i < options.size(); i++) { - if (options[i].val > 1024) { - fprintf(stderr, "using -x %s would save about %lld, for a tile size of of %lld\n", options[i].name, options[i].val, orig - options[i].val); - } - } - - struct pool keys, values; - pool_init(&keys, 0); - pool_init(&values, 0); - long long count = 0; - - std::vector empty; - mapnik::vector::tile tile = create_tile(layername, line_detail, features, &count, &keys, &values, nlayers); - - std::string s; - std::string compressed; - - tile.SerializeToString(&s); - compress(s, compressed); - fprintf(stderr, "geometry alone (-X) would be %lld\n", (long long) compressed.size()); - - pool_free(&values); - pool_free(&keys); -} -#endif - -void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom) { +void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom) { if (geom.size() > 0 && nextzoom <= file_maxzoom) { int xo, yo; int span = 1 << (nextzoom - z); @@ -405,7 +353,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb // j is the shard that the child tile's data is being written to. // - // Be careful: We can't jump more zoom levels than MAX_ZOOM_INCREMENT + // Be careful: We can't jump more zoom levels than max_zoom_increment // because that could break the constraint that each of the children // of the current tile must have its own shard, because the data for // the child tile must be contiguous within the shard. @@ -414,9 +362,14 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb // the four that would normally result from splitting one tile, // because it will go through all the shards when it does the // next zoom. + // + // If child_shards is a power of 2 but not a power of 4, this will + // shard X more widely than Y. XXX Is there a better way to do this + // without causing collisions? - int j = ((jx & ((1 << MAX_ZOOM_INCREMENT) - 1)) << MAX_ZOOM_INCREMENT) | - ((jy & ((1 << MAX_ZOOM_INCREMENT) - 1))); + int j = ((jx << max_zoom_increment) | + ((jy & ((1 << max_zoom_increment) - 1)))) & + (child_shards - 1); { if (!within[j]) { @@ -465,20 +418,29 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb } } -long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional) { +long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, char *prevent, char *additional, int child_shards) { int line_detail; - static bool evaluated = false; - double oprogress = 0; double fraction = 1; char *og = *geoms; + // XXX is there a way to do this without floating point? + int max_zoom_increment = log(child_shards) / log(4); + if (child_shards < 4 || max_zoom_increment < 1) { + fprintf(stderr, "Internal error: %d shards, max zoom increment %d\n", child_shards, max_zoom_increment); + exit(EXIT_FAILURE); + } + if ((((child_shards - 1) << 1) & child_shards) != child_shards) { + fprintf(stderr, "Internal error: %d shards not a power of 2\n", child_shards); + exit(EXIT_FAILURE); + } + int nextzoom = z + 1; if (nextzoom < file_minzoom) { - if (z + MAX_ZOOM_INCREMENT > file_minzoom) { + if (z + max_zoom_increment > file_minzoom) { nextzoom = file_minzoom; } else { - nextzoom = z + MAX_ZOOM_INCREMENT; + nextzoom = z + max_zoom_increment; } } @@ -497,7 +459,6 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } long long count = 0; - // long long along = 0; double accum_area = 0; double interval = 0; @@ -520,8 +481,12 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f features.push_back(std::vector()); } - int within[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0}; - long long geompos[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0}; + int within[child_shards]; + long long geompos[child_shards]; + memset(within, '\0', sizeof(within)); + memset(geompos, '\0', sizeof(geompos)); + + double oprogress = 0; *geoms = og; @@ -556,8 +521,8 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f signed char feature_minzoom; deserialize_byte(geoms, &feature_minzoom); - double progress = floor((((*geoms - geomstart + along) / (double) todo) + z) / (file_maxzoom + 1) * 1000) / 10; - if (progress != oprogress) { + double progress = floor((((*geoms - geomstart + *along) / (double) todo) + z) / (file_maxzoom + 1) * 1000) / 10; + if (progress >= oprogress + 0.1) { if (!quiet) { fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); } @@ -594,7 +559,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f } if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */ - rewrite(geom, z, nextzoom, file_maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom); + rewrite(geom, z, nextzoom, file_maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, child_shards, max_zoom_increment, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom); } if (z < file_minzoom) { @@ -716,13 +681,13 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f c.coalesced = false; c.original_seq = original_seq; - decode_meta(&meta, stringpool, keys[layer], values[layer], file_keys[layer], &c.meta, NULL); + decode_meta(&meta, stringpool, keys[layer], values[layer], file_keys[layer], &c.meta); features[layer].push_back(c); } } int j; - for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { + for (j = 0; j < child_shards; j++) { if (within[j]) { serialize_byte(geomfile[j], -2, &geompos[j], fname); within[j] = 0; @@ -805,13 +770,6 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >500000 \n", z, tx, ty, (long long) compressed.size(), line_detail); } - if (line_detail == min_detail || !evaluated) { - evaluated = true; -#if 0 - evaluate(features[0], metabase, file_keys[0], layername, line_detail, compressed.size()); // XXX layer -#endif - } - if (prevent['d' & 0xFF]) { // The 95% is a guess to avoid too many retries // and probably actually varies based on how much duplicated metadata there is @@ -823,7 +781,18 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f line_detail++; // to keep it the same when the loop decrements it } } else { + if (pthread_mutex_lock(&db_lock) != 0) { + perror("pthread_mutex_lock"); + exit(EXIT_FAILURE); + } + mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); + + if (pthread_mutex_unlock(&db_lock) != 0) { + perror("pthread_mutex_unlock"); + exit(EXIT_FAILURE); + } + return count; } } else { @@ -841,15 +810,126 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f return -1; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail) { +struct task { + int fileno; + struct task *next; +} tasks[TEMP_FILES]; + +struct write_tile_args { + struct task *tasks; + char *metabase; + char *stringpool; + int min_detail; + int basezoom; + struct pool **file_keys; + char **layernames; + sqlite3 *outdb; + double droprate; + int buffer; + const char *fname; + FILE **geomfile; + int file_minzoom; + int file_maxzoom; + double todo; + volatile long long *along; + double gamma; + int nlayers; + char *prevent; + char *additional; + int child_shards; + int *geomfd; + off_t *geom_size; + volatile unsigned *midx; + volatile unsigned *midy; + int maxzoom; + int minzoom; + int full_detail; + int low_detail; + volatile long long *most; +}; + +void *run_thread(void *vargs) { + write_tile_args *arg = (write_tile_args *) vargs; + struct task *task; + + for (task = arg->tasks; task != NULL; task = task->next) { + int j = task->fileno; + + if (arg->geomfd[j] < 0) { + // only one source file for zoom level 0 + continue; + } + if (arg->geom_size[j] == 0) { + continue; + } + + // printf("%lld of geom_size\n", (long long) geom_size[j]); + + char *geom = (char *) mmap(NULL, arg->geom_size[j], PROT_READ, MAP_PRIVATE, arg->geomfd[j], 0); + if (geom == MAP_FAILED) { + perror("mmap geom"); + exit(EXIT_FAILURE); + } + + char *geomstart = geom; + char *end = geom + arg->geom_size[j]; + char *prevgeom = geom; + + while (geom < end) { + int z; + unsigned x, y; + + deserialize_int(&geom, &z); + deserialize_uint(&geom, &x); + deserialize_uint(&geom, &y); + + // fprintf(stderr, "%d/%u/%u\n", z, x, y); + + long long len = write_tile(&geom, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->maxzoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards); + + if (len < 0) { + int *err = (int *) malloc(sizeof(int)); + *err = z - 1; + return err; + } + + if (pthread_mutex_lock(&var_lock) != 0) { + perror("pthread_mutex_lock"); + exit(EXIT_FAILURE); + } + + if (z == arg->maxzoom && len > *arg->most) { + *arg->midx = x; + *arg->midy = y; + *arg->most = len; + } + + *arg->along += geom - prevgeom; + prevgeom = geom; + + if (pthread_mutex_unlock(&var_lock) != 0) { + perror("pthread_mutex_unlock"); + exit(EXIT_FAILURE); + } + } + + if (munmap(geomstart, arg->geom_size[j]) != 0) { + perror("munmap geom"); + } + } + + return NULL; +} + +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail) { int i; for (i = 0; i <= maxzoom; i++) { long long most = 0; - FILE *sub[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; - int subfd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; + FILE *sub[TEMP_FILES]; + int subfd[TEMP_FILES]; int j; - for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { + for (j = 0; j < TEMP_FILES; j++) { char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX" XSTRINGIFY(INT_MAX)) + 1]; sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j); subfd[j] = mkstemp(geomname); @@ -866,62 +946,132 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo unlink(geomname); } + int useful_threads = 0; long long todo = 0; long long along = 0; - for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { + for (j = 0; j < TEMP_FILES; j++) { todo += geom_size[j]; + if (geom_size[j] > 0) { + useful_threads++; + } } - for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { - if (geomfd[j] < 0) { - // only one source file for zoom level 0 - continue; +#define MAX_THREADS 20 // XXX Obtain from sysctl(hw.ncpu), /proc/cpuinfo, etc. + + int threads = MAX_THREADS; + if (threads > TEMP_FILES / 4) { + threads = TEMP_FILES / 4; + } + // XXX is it useful to divide further if we know we are skipping + // some zoom levels? Is it faster to have fewer CPUs working on + // sharding, but more deeply, or fewer CPUs, less deeply? + if (threads > useful_threads) { + threads = useful_threads; + } + // Round down to a power of 2 + threads = 1 << (int) (log(threads) / log(2)); + + // Assign temporary files to threads + + struct dispatch { + struct task *tasks; + long long todo; + struct dispatch *next; + } dispatches[threads]; + struct dispatch *dispatch_head = &dispatches[0]; + for (j = 0; j < threads; j++) { + dispatches[j].tasks = NULL; + dispatches[j].todo = 0; + if (j + 1 < threads) { + dispatches[j].next = &dispatches[j + 1]; + } else { + dispatches[j].next = NULL; } + } + + for (j = 0; j < TEMP_FILES; j++) { if (geom_size[j] == 0) { continue; } - // printf("%lld of geom_size\n", (long long) geom_size[j]); + tasks[j].fileno = j; + tasks[j].next = dispatch_head->tasks; + dispatch_head->tasks = &tasks[j]; + dispatch_head->todo += geom_size[j]; - char *geom = (char *) mmap(NULL, geom_size[j], PROT_READ, MAP_PRIVATE, geomfd[j], 0); - if (geom == MAP_FAILED) { - perror("mmap geom"); - exit(EXIT_FAILURE); - } + struct dispatch *here = dispatch_head; + dispatch_head = dispatch_head->next; - char *geomstart = geom; - char *end = geom + geom_size[j]; - - while (geom < end) { - int z; - unsigned x, y; - - deserialize_int(&geom, &z); - deserialize_uint(&geom, &x); - deserialize_uint(&geom, &y); - - // fprintf(stderr, "%d/%u/%u\n", z, x, y); - - long long len = write_tile(&geom, metabase, stringpool, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, min_detail, maxzoom, file_keys, layernames, outdb, droprate, buffer, fname, sub, minzoom, maxzoom, todo, geomstart, along, gamma, nlayers, prevent, additional); - - if (len < 0) { - return i - 1; - } - - if (z == maxzoom && len > most) { - *midx = x; - *midy = y; - most = len; + dispatch **d; + for (d = &dispatch_head; *d != NULL; d = &((*d)->next)) { + if (here->todo < (*d)->todo) { + break; } } - if (munmap(geomstart, geom_size[j]) != 0) { - perror("munmap geom"); - } - along += geom_size[j]; + here->next = *d; + *d = here; } - for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { + pthread_t pthreads[threads]; + write_tile_args args[threads]; + + int thread; + for (thread = 0; thread < threads; thread++) { + args[thread].metabase = metabase; + args[thread].stringpool = stringpool; + args[thread].min_detail = min_detail; + args[thread].basezoom = maxzoom; // XXX rename? + args[thread].file_keys = file_keys; // locked with var_lock + args[thread].layernames = layernames; + args[thread].outdb = outdb; // locked with db_lock + args[thread].droprate = droprate; + args[thread].buffer = buffer; + args[thread].fname = fname; + args[thread].geomfile = sub + thread * (TEMP_FILES / threads); + args[thread].file_minzoom = minzoom; + args[thread].file_maxzoom = maxzoom; + args[thread].todo = todo; + args[thread].along = &along; // locked with var_lock + args[thread].gamma = gamma; + args[thread].nlayers = nlayers; + args[thread].prevent = prevent; + args[thread].additional = additional; + args[thread].child_shards = TEMP_FILES / threads; + + args[thread].geomfd = geomfd; + args[thread].geom_size = geom_size; + args[thread].midx = midx; // locked with var_lock + args[thread].midy = midy; // locked with var_lock + args[thread].maxzoom = maxzoom; + args[thread].minzoom = minzoom; + args[thread].full_detail = full_detail; + args[thread].low_detail = low_detail; + args[thread].most = &most; // locked with var_lock + + args[thread].tasks = dispatches[thread].tasks; + + if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) { + perror("pthread_create"); + exit(EXIT_FAILURE); + } + } + + int err = INT_MAX; + + for (thread = 0; thread < threads; thread++) { + void *retval; + + if (pthread_join(pthreads[thread], &retval) != 0) { + perror("pthread_join"); + } + + if (retval != NULL) { + err = *((int *) retval); + } + } + + for (j = 0; j < TEMP_FILES; j++) { close(geomfd[j]); fclose(sub[j]); @@ -934,6 +1084,10 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo geomfd[j] = subfd[j]; geom_size[j] = geomst.st_size; } + + if (err != INT_MAX) { + return err; + } } if (!quiet) { diff --git a/tile.h b/tile.h index 3094fe63..8d73566f 100644 --- a/tile.h +++ b/tile.h @@ -27,10 +27,10 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type); long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail); extern unsigned initial_x, initial_y; extern int geometry_scale; extern int quiet; -#define MAX_ZOOM_INCREMENT 3 +#define TEMP_FILES 64 diff --git a/version.h b/version.h index 3dfed500..25399db1 100644 --- a/version.h +++ b/version.h @@ -1 +1 @@ -#define VERSION "tippecanoe v1.2.0\n" +#define VERSION "tippecanoe v1.3.0\n"