diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-05 09:55:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #100 from mapbox/multithread

Browse Source 
Multithreaded tile generation
This commit is contained in:
Eric Fischer
2015-10-20 15:21:46 -07:00
parent 89e1640496 e554a121fb
commit 3fbd0d6546
9 changed files with 330 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+13
View File
@@ -1,3 +1,16 @@
## 1.3.0
* Tile generation is multithreaded to take advantage of multiple CPUs
* More compact data representation reduces memory usage and improves speed
* Polygon clipping uses [Clipper](http://www.angusj.com/delphi/clipper/documentation/Docs/_Body.htm)
and makes sure interior and exterior rings are distinguished by winding order
* Individual GeoJSON features can specify their own minzoom and maxzoom
* New `tile-join` utility can add new properties from a CSV file to an existing tileset
* Feature coalescing, line-reversing, and reordering by attribute are now options, not defaults
* Output of `decode` utility is now in GeoJSON format
* Tile generation with a minzoom spends less time on unused lower zoom levels
* Bare geometries without a Feature wrapper are accepted
## 1.2.0 ## 1.2.0
* Switched to top-down rendering, yielding performance improvements * Switched to top-down rendering, yielding performance improvements
Makefile
+1 -1
View File
@@ -25,7 +25,7 @@ INCLUDES = -I/usr/local/include
LIBS = -L/usr/local/lib LIBS = -L/usr/local/lib
tippecanoe: geojson.o jsonpull.o vector_tile.pb.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o tippecanoe: geojson.o jsonpull.o vector_tile.pb.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o
g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 -lpthread
enumerate: enumerate.o enumerate: enumerate.o
gcc $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lsqlite3 gcc $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lsqlite3
decode.cc
-1
View File
@@ -417,6 +417,5 @@ int main(int argc, char **argv) {
usage(argv); usage(argv);
} }
return 0; return 0;
} }
geojson.c
+4 -4
View File
@@ -1073,14 +1073,14 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
int fd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; int fd[TEMP_FILES];
off_t size[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; off_t size[TEMP_FILES];
fd[0] = geomfd; fd[0] = geomfd;
size[0] = geomst.st_size; size[0] = geomst.st_size;
int j; int j;
for (j = 1; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { for (j = 1; j < TEMP_FILES; j++) {
fd[j] = -1; fd[j] = -1;
size[j] = 0; size[j] = 0;
} }
@@ -1089,7 +1089,7 @@ int read_json(int argc, char **argv, char *fname, const char *layername, int max
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", seq, (long long) geomst.st_size, (long long) metast.st_size, poolfile->off); fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", seq, (long long) geomst.st_size, (long long) metast.st_size, poolfile->off);
} }
int written = traverse_zooms(fd, size, meta, stringpool, file_bbox, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail); int written = traverse_zooms(fd, size, meta, stringpool, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail);
if (maxzoom != written) { if (maxzoom != written) {
fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written);
geometry.cc
+5 -5
View File
@@ -339,11 +339,11 @@ drawvec reduce_tiny_poly(drawvec &geom, int z, int detail, bool *reduced, double
if (*accum_area > pixel * pixel) { if (*accum_area > pixel * pixel) {
// XXX use centroid; // XXX use centroid;
out.push_back(draw(VT_MOVETO, geom[i].x - pixel/2, geom[i].y - pixel/2)); out.push_back(draw(VT_MOVETO, geom[i].x - pixel / 2, geom[i].y - pixel / 2));
out.push_back(draw(VT_LINETO, geom[i].x + pixel/2, geom[i].y - pixel/2)); out.push_back(draw(VT_LINETO, geom[i].x + pixel / 2, geom[i].y - pixel / 2));
out.push_back(draw(VT_LINETO, geom[i].x + pixel/2, geom[i].y + pixel/2)); out.push_back(draw(VT_LINETO, geom[i].x + pixel / 2, geom[i].y + pixel / 2));
out.push_back(draw(VT_LINETO, geom[i].x - pixel/2, geom[i].y + pixel/2)); out.push_back(draw(VT_LINETO, geom[i].x - pixel / 2, geom[i].y + pixel / 2));
out.push_back(draw(VT_LINETO, geom[i].x - pixel/2, geom[i].y - pixel/2)); out.push_back(draw(VT_LINETO, geom[i].x - pixel / 2, geom[i].y - pixel / 2));
*accum_area -= pixel * pixel; *accum_area -= pixel * pixel;
} }
pool.c
+12 -11
View File
@@ -34,26 +34,27 @@ struct pool_val *pool(struct pool *p, const char *s, int type) {
} }
} }
*v = malloc(sizeof(struct pool_val)); struct pool_val *nv = malloc(sizeof(struct pool_val));
if (*v == NULL) { if (nv == NULL) {
fprintf(stderr, "out of memory making string pool\n"); fprintf(stderr, "out of memory making string pool\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
(*v)->left = NULL; nv->left = NULL;
(*v)->right = NULL; nv->right = NULL;
(*v)->next = NULL; nv->next = NULL;
(*v)->s = s; nv->s = s;
(*v)->type = type; nv->type = type;
(*v)->n = p->n++; nv->n = p->n++;
if (p->tail != NULL) { if (p->tail != NULL) {
p->tail->next = *v; p->tail->next = nv;
} }
p->tail = *v; p->tail = nv;
if (p->head == NULL) { if (p->head == NULL) {
p->head = *v; p->head = nv;
} }
*v = nv;
return *v; return *v;
} }
tile.cc
+292 -138
View File
@@ -15,6 +15,7 @@
#include <sys/mman.h> #include <sys/mman.h>
#include <math.h> #include <math.h>
#include <sqlite3.h> #include <sqlite3.h>
#include <pthread.h>
#include "vector_tile.pb.h" #include "vector_tile.pb.h"
#include "geometry.hh" #include "geometry.hh"
@@ -31,6 +32,9 @@ extern "C" {
#define XSTRINGIFY(s) STRINGIFY(s) #define XSTRINGIFY(s) STRINGIFY(s)
#define STRINGIFY(s) #s #define STRINGIFY(s) #s
pthread_mutex_t db_lock = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t var_lock = PTHREAD_MUTEX_INITIALIZER;
// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
static inline int compress(std::string const &input, std::string &output) { static inline int compress(std::string const &input, std::string &output) {
z_stream deflate_s; z_stream deflate_s;
@@ -208,26 +212,30 @@ struct pool_val *retrieve_string(char **f, struct pool *p, char *stringpool) {
return ret; return ret;
} }
void decode_meta(char **meta, char *stringpool, struct pool *keys, struct pool *values, struct pool *file_keys, std::vector<int> *intmeta, char *only) { void decode_meta(char **meta, char *stringpool, struct pool *keys, struct pool *values, struct pool *file_keys, std::vector<int> *intmeta) {
int m; int m;
deserialize_int(meta, &m); deserialize_int(meta, &m);
int i; int i;
for (i = 0; i < m; i++) { for (i = 0; i < m; i++) {
struct pool_val *key = retrieve_string(meta, keys, stringpool); struct pool_val *key = retrieve_string(meta, keys, stringpool);
struct pool_val *value = retrieve_string(meta, values, stringpool);
if (only != NULL && (strcmp(key->s, only) != 0)) { intmeta->push_back(key->n);
// XXX if evaluate ever works again, check whether this is sufficient intmeta->push_back(value->n);
(void) retrieve_string(meta, values, stringpool);
} else {
struct pool_val *value = retrieve_string(meta, values, stringpool);
intmeta->push_back(key->n); if (!is_pooled(file_keys, key->s, value->type)) {
intmeta->push_back(value->n); if (pthread_mutex_lock(&var_lock) != 0) {
perror("pthread_mutex_lock");
exit(EXIT_FAILURE);
}
if (!is_pooled(file_keys, key->s, value->type)) { // Dup to retain after munmap
// Dup to retain after munmap pool(file_keys, strdup(key->s), value->type);
pool(file_keys, strdup(key->s), value->type);
if (pthread_mutex_unlock(&var_lock) != 0) {
perror("pthread_mutex_unlock");
exit(EXIT_FAILURE);
} }
} }
} }
@@ -309,67 +317,7 @@ struct sll {
} }
}; };
#if 0 void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom) {
void evaluate(std::vector<coalesce> &features, char *metabase, struct pool *file_keys, const char *layername, int line_detail, long long orig) {
std::vector<sll> options;
struct pool_val *pv;
for (pv = file_keys->head; pv != NULL; pv = pv->next) {
struct pool keys, values;
pool_init(&keys, 0);
pool_init(&values, 0);
long long count = 0;
for (unsigned i = 0; i < features.size(); i++) {
char *meta = features[i].metasrc;
features[i].meta.resize(0);
decode_meta(&meta, &keys, &values, file_keys, &features[i].meta, pv->s);
}
std::vector<coalesce> empty;
mapnik::vector::tile tile = create_tile(layername, line_detail, empty, &count, &keys, &values, 1); // XXX layer
std::string s;
std::string compressed;
tile.SerializeToString(&s);
compress(s, compressed);
options.push_back(sll(pv->s, compressed.size()));
pool_free(&values);
pool_free(&keys);
}
std::sort(options.begin(), options.end());
for (unsigned i = 0; i < options.size(); i++) {
if (options[i].val > 1024) {
fprintf(stderr, "using -x %s would save about %lld, for a tile size of of %lld\n", options[i].name, options[i].val, orig - options[i].val);
}
}
struct pool keys, values;
pool_init(&keys, 0);
pool_init(&values, 0);
long long count = 0;
std::vector<coalesce> empty;
mapnik::vector::tile tile = create_tile(layername, line_detail, features, &count, &keys, &values, nlayers);
std::string s;
std::string compressed;
tile.SerializeToString(&s);
compress(s, compressed);
fprintf(stderr, "geometry alone (-X) would be %lld\n", (long long) compressed.size());
pool_free(&values);
pool_free(&keys);
}
#endif
void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom) {
if (geom.size() > 0 && nextzoom <= file_maxzoom) { if (geom.size() > 0 && nextzoom <= file_maxzoom) {
int xo, yo; int xo, yo;
int span = 1 << (nextzoom - z); int span = 1 << (nextzoom - z);
@@ -405,7 +353,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb
// j is the shard that the child tile's data is being written to. // j is the shard that the child tile's data is being written to.
// //
// Be careful: We can't jump more zoom levels than MAX_ZOOM_INCREMENT // Be careful: We can't jump more zoom levels than max_zoom_increment
// because that could break the constraint that each of the children // because that could break the constraint that each of the children
// of the current tile must have its own shard, because the data for // of the current tile must have its own shard, because the data for
// the child tile must be contiguous within the shard. // the child tile must be contiguous within the shard.
@@ -414,9 +362,14 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb
// the four that would normally result from splitting one tile, // the four that would normally result from splitting one tile,
// because it will go through all the shards when it does the // because it will go through all the shards when it does the
// next zoom. // next zoom.
//
// If child_shards is a power of 2 but not a power of 4, this will
// shard X more widely than Y. XXX Is there a better way to do this
// without causing collisions?
int j = ((jx & ((1 << MAX_ZOOM_INCREMENT) - 1)) << MAX_ZOOM_INCREMENT) | int j = ((jx << max_zoom_increment) |
((jy & ((1 << MAX_ZOOM_INCREMENT) - 1))); ((jy & ((1 << max_zoom_increment) - 1)))) &
(child_shards - 1);
{ {
if (!within[j]) { if (!within[j]) {
@@ -465,20 +418,29 @@ void rewrite(drawvec &geom, int z, int nextzoom, int file_maxzoom, long long *bb
} }
} }
long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional) { long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, volatile long long *along, double gamma, int nlayers, char *prevent, char *additional, int child_shards) {
int line_detail; int line_detail;
static bool evaluated = false;
double oprogress = 0;
double fraction = 1; double fraction = 1;
char *og = *geoms; char *og = *geoms;
// XXX is there a way to do this without floating point?
int max_zoom_increment = log(child_shards) / log(4);
if (child_shards < 4 || max_zoom_increment < 1) {
fprintf(stderr, "Internal error: %d shards, max zoom increment %d\n", child_shards, max_zoom_increment);
exit(EXIT_FAILURE);
}
if ((((child_shards - 1) << 1) & child_shards) != child_shards) {
fprintf(stderr, "Internal error: %d shards not a power of 2\n", child_shards);
exit(EXIT_FAILURE);
}
int nextzoom = z + 1; int nextzoom = z + 1;
if (nextzoom < file_minzoom) { if (nextzoom < file_minzoom) {
if (z + MAX_ZOOM_INCREMENT > file_minzoom) { if (z + max_zoom_increment > file_minzoom) {
nextzoom = file_minzoom; nextzoom = file_minzoom;
} else { } else {
nextzoom = z + MAX_ZOOM_INCREMENT; nextzoom = z + max_zoom_increment;
} }
} }
@@ -497,7 +459,6 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
} }
long long count = 0; long long count = 0;
// long long along = 0;
double accum_area = 0; double accum_area = 0;
double interval = 0; double interval = 0;
@@ -520,8 +481,12 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
features.push_back(std::vector<coalesce>()); features.push_back(std::vector<coalesce>());
} }
int within[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0}; int within[child_shards];
long long geompos[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)] = {0}; long long geompos[child_shards];
memset(within, '\0', sizeof(within));
memset(geompos, '\0', sizeof(geompos));
double oprogress = 0;
*geoms = og; *geoms = og;
@@ -556,8 +521,8 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
signed char feature_minzoom; signed char feature_minzoom;
deserialize_byte(geoms, &feature_minzoom); deserialize_byte(geoms, &feature_minzoom);
double progress = floor((((*geoms - geomstart + along) / (double) todo) + z) / (file_maxzoom + 1) * 1000) / 10; double progress = floor((((*geoms - geomstart + *along) / (double) todo) + z) / (file_maxzoom + 1) * 1000) / 10;
if (progress != oprogress) { if (progress >= oprogress + 0.1) {
if (!quiet) { if (!quiet) {
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
} }
@@ -594,7 +559,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
} }
if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */ if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */
rewrite(geom, z, nextzoom, file_maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom); rewrite(geom, z, nextzoom, file_maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, child_shards, max_zoom_increment, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom);
} }
if (z < file_minzoom) { if (z < file_minzoom) {
@@ -716,13 +681,13 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
c.coalesced = false; c.coalesced = false;
c.original_seq = original_seq; c.original_seq = original_seq;
decode_meta(&meta, stringpool, keys[layer], values[layer], file_keys[layer], &c.meta, NULL); decode_meta(&meta, stringpool, keys[layer], values[layer], file_keys[layer], &c.meta);
features[layer].push_back(c); features[layer].push_back(c);
} }
} }
int j; int j;
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { for (j = 0; j < child_shards; j++) {
if (within[j]) { if (within[j]) {
serialize_byte(geomfile[j], -2, &geompos[j], fname); serialize_byte(geomfile[j], -2, &geompos[j], fname);
within[j] = 0; within[j] = 0;
@@ -805,13 +770,6 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >500000 \n", z, tx, ty, (long long) compressed.size(), line_detail); fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >500000 \n", z, tx, ty, (long long) compressed.size(), line_detail);
} }
if (line_detail == min_detail || !evaluated) {
evaluated = true;
#if 0
evaluate(features[0], metabase, file_keys[0], layername, line_detail, compressed.size()); // XXX layer
#endif
}
if (prevent['d' & 0xFF]) { if (prevent['d' & 0xFF]) {
// The 95% is a guess to avoid too many retries // The 95% is a guess to avoid too many retries
// and probably actually varies based on how much duplicated metadata there is // and probably actually varies based on how much duplicated metadata there is
@@ -823,7 +781,18 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
line_detail++; // to keep it the same when the loop decrements it line_detail++; // to keep it the same when the loop decrements it
} }
} else { } else {
if (pthread_mutex_lock(&db_lock) != 0) {
perror("pthread_mutex_lock");
exit(EXIT_FAILURE);
}
mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
if (pthread_mutex_unlock(&db_lock) != 0) {
perror("pthread_mutex_unlock");
exit(EXIT_FAILURE);
}
return count; return count;
} }
} else { } else {
@@ -841,15 +810,126 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, unsigned *f
return -1; return -1;
} }
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail) { struct task {
int fileno;
struct task *next;
} tasks[TEMP_FILES];
struct write_tile_args {
struct task *tasks;
char *metabase;
char *stringpool;
int min_detail;
int basezoom;
struct pool **file_keys;
char **layernames;
sqlite3 *outdb;
double droprate;
int buffer;
const char *fname;
FILE **geomfile;
int file_minzoom;
int file_maxzoom;
double todo;
volatile long long *along;
double gamma;
int nlayers;
char *prevent;
char *additional;
int child_shards;
int *geomfd;
off_t *geom_size;
volatile unsigned *midx;
volatile unsigned *midy;
int maxzoom;
int minzoom;
int full_detail;
int low_detail;
volatile long long *most;
};
void *run_thread(void *vargs) {
write_tile_args *arg = (write_tile_args *) vargs;
struct task *task;
for (task = arg->tasks; task != NULL; task = task->next) {
int j = task->fileno;
if (arg->geomfd[j] < 0) {
// only one source file for zoom level 0
continue;
}
if (arg->geom_size[j] == 0) {
continue;
}
// printf("%lld of geom_size\n", (long long) geom_size[j]);
char *geom = (char *) mmap(NULL, arg->geom_size[j], PROT_READ, MAP_PRIVATE, arg->geomfd[j], 0);
if (geom == MAP_FAILED) {
perror("mmap geom");
exit(EXIT_FAILURE);
}
char *geomstart = geom;
char *end = geom + arg->geom_size[j];
char *prevgeom = geom;
while (geom < end) {
int z;
unsigned x, y;
deserialize_int(&geom, &z);
deserialize_uint(&geom, &x);
deserialize_uint(&geom, &y);
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
long long len = write_tile(&geom, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->maxzoom, arg->file_keys, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, geomstart, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards);
if (len < 0) {
int *err = (int *) malloc(sizeof(int));
*err = z - 1;
return err;
}
if (pthread_mutex_lock(&var_lock) != 0) {
perror("pthread_mutex_lock");
exit(EXIT_FAILURE);
}
if (z == arg->maxzoom && len > *arg->most) {
*arg->midx = x;
*arg->midy = y;
*arg->most = len;
}
*arg->along += geom - prevgeom;
prevgeom = geom;
if (pthread_mutex_unlock(&var_lock) != 0) {
perror("pthread_mutex_unlock");
exit(EXIT_FAILURE);
}
}
if (munmap(geomstart, arg->geom_size[j]) != 0) {
perror("munmap geom");
}
}
return NULL;
}
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail) {
int i; int i;
for (i = 0; i <= maxzoom; i++) { for (i = 0; i <= maxzoom; i++) {
long long most = 0; long long most = 0;
FILE *sub[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; FILE *sub[TEMP_FILES];
int subfd[(1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT)]; int subfd[TEMP_FILES];
int j; int j;
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { for (j = 0; j < TEMP_FILES; j++) {
char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX" XSTRINGIFY(INT_MAX)) + 1]; char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX" XSTRINGIFY(INT_MAX)) + 1];
sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j); sprintf(geomname, "%s/geom%d.XXXXXXXX", tmpdir, j);
subfd[j] = mkstemp(geomname); subfd[j] = mkstemp(geomname);
@@ -866,62 +946,132 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
unlink(geomname); unlink(geomname);
} }
int useful_threads = 0;
long long todo = 0; long long todo = 0;
long long along = 0; long long along = 0;
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { for (j = 0; j < TEMP_FILES; j++) {
todo += geom_size[j]; todo += geom_size[j];
if (geom_size[j] > 0) {
useful_threads++;
}
} }
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { #define MAX_THREADS 20 // XXX Obtain from sysctl(hw.ncpu), /proc/cpuinfo, etc.
if (geomfd[j] < 0) {
// only one source file for zoom level 0 int threads = MAX_THREADS;
continue; if (threads > TEMP_FILES / 4) {
threads = TEMP_FILES / 4;
}
// XXX is it useful to divide further if we know we are skipping
// some zoom levels? Is it faster to have fewer CPUs working on
// sharding, but more deeply, or fewer CPUs, less deeply?
if (threads > useful_threads) {
threads = useful_threads;
}
// Round down to a power of 2
threads = 1 << (int) (log(threads) / log(2));
// Assign temporary files to threads
struct dispatch {
struct task *tasks;
long long todo;
struct dispatch *next;
} dispatches[threads];
struct dispatch *dispatch_head = &dispatches[0];
for (j = 0; j < threads; j++) {
dispatches[j].tasks = NULL;
dispatches[j].todo = 0;
if (j + 1 < threads) {
dispatches[j].next = &dispatches[j + 1];
} else {
dispatches[j].next = NULL;
} }
}
for (j = 0; j < TEMP_FILES; j++) {
if (geom_size[j] == 0) { if (geom_size[j] == 0) {
continue; continue;
} }
// printf("%lld of geom_size\n", (long long) geom_size[j]); tasks[j].fileno = j;
tasks[j].next = dispatch_head->tasks;
dispatch_head->tasks = &tasks[j];
dispatch_head->todo += geom_size[j];
char *geom = (char *) mmap(NULL, geom_size[j], PROT_READ, MAP_PRIVATE, geomfd[j], 0); struct dispatch *here = dispatch_head;
if (geom == MAP_FAILED) { dispatch_head = dispatch_head->next;
perror("mmap geom");
exit(EXIT_FAILURE);
}
char *geomstart = geom; dispatch **d;
char *end = geom + geom_size[j]; for (d = &dispatch_head; *d != NULL; d = &((*d)->next)) {
if (here->todo < (*d)->todo) {
while (geom < end) { break;
int z;
unsigned x, y;
deserialize_int(&geom, &z);
deserialize_uint(&geom, &x);
deserialize_uint(&geom, &y);
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
long long len = write_tile(&geom, metabase, stringpool, file_bbox, z, x, y, z == maxzoom ? full_detail : low_detail, min_detail, maxzoom, file_keys, layernames, outdb, droprate, buffer, fname, sub, minzoom, maxzoom, todo, geomstart, along, gamma, nlayers, prevent, additional);
if (len < 0) {
return i - 1;
}
if (z == maxzoom && len > most) {
*midx = x;
*midy = y;
most = len;
} }
} }
if (munmap(geomstart, geom_size[j]) != 0) { here->next = *d;
perror("munmap geom"); *d = here;
}
along += geom_size[j];
} }
for (j = 0; j < (1 << MAX_ZOOM_INCREMENT) * (1 << MAX_ZOOM_INCREMENT); j++) { pthread_t pthreads[threads];
write_tile_args args[threads];
int thread;
for (thread = 0; thread < threads; thread++) {
args[thread].metabase = metabase;
args[thread].stringpool = stringpool;
args[thread].min_detail = min_detail;
args[thread].basezoom = maxzoom; // XXX rename?
args[thread].file_keys = file_keys; // locked with var_lock
args[thread].layernames = layernames;
args[thread].outdb = outdb; // locked with db_lock
args[thread].droprate = droprate;
args[thread].buffer = buffer;
args[thread].fname = fname;
args[thread].geomfile = sub + thread * (TEMP_FILES / threads);
args[thread].file_minzoom = minzoom;
args[thread].file_maxzoom = maxzoom;
args[thread].todo = todo;
args[thread].along = &along; // locked with var_lock
args[thread].gamma = gamma;
args[thread].nlayers = nlayers;
args[thread].prevent = prevent;
args[thread].additional = additional;
args[thread].child_shards = TEMP_FILES / threads;
args[thread].geomfd = geomfd;
args[thread].geom_size = geom_size;
args[thread].midx = midx; // locked with var_lock
args[thread].midy = midy; // locked with var_lock
args[thread].maxzoom = maxzoom;
args[thread].minzoom = minzoom;
args[thread].full_detail = full_detail;
args[thread].low_detail = low_detail;
args[thread].most = &most; // locked with var_lock
args[thread].tasks = dispatches[thread].tasks;
if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) {
perror("pthread_create");
exit(EXIT_FAILURE);
}
}
int err = INT_MAX;
for (thread = 0; thread < threads; thread++) {
void *retval;
if (pthread_join(pthreads[thread], &retval) != 0) {
perror("pthread_join");
}
if (retval != NULL) {
err = *((int *) retval);
}
}
for (j = 0; j < TEMP_FILES; j++) {
close(geomfd[j]); close(geomfd[j]);
fclose(sub[j]); fclose(sub[j]);
@@ -934,6 +1084,10 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
geomfd[j] = subfd[j]; geomfd[j] = subfd[j];
geom_size[j] = geomst.st_size; geom_size[j] = geomst.st_size;
} }
if (err != INT_MAX) {
return err;
}
} }
if (!quiet) { if (!quiet) {
tile.h
+2 -2
View File
@@ -27,10 +27,10 @@ struct pool_val *deserialize_string(char **f, struct pool *p, int type);
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional); long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, struct pool **file_keys, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, char *prevent, char *additional);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *file_bbox, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail); int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, struct pool **file_keys, unsigned *midx, unsigned *midy, char **layernames, int maxzoom, int minzoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, char *prevent, char *additional, int full_detail, int low_detail, int min_detail);
extern unsigned initial_x, initial_y; extern unsigned initial_x, initial_y;
extern int geometry_scale; extern int geometry_scale;
extern int quiet; extern int quiet;
#define MAX_ZOOM_INCREMENT 3 #define TEMP_FILES 64
version.h
+1 -1
View File
@@ -1 +1 @@
#define VERSION "tippecanoe v1.2.0\n" #define VERSION "tippecanoe v1.3.0\n"