diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-05 18:05:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Handle nonzero initial zooms

Browse Source
This commit is contained in:
Erica Fischer
2023-02-01 13:32:58 -08:00
parent 1b176e55c3
commit d271cf233d
3 changed files with 20 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+2 -2
View File
@@ -1947,7 +1947,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
std::atomic<long long> geompos(0); std::atomic<long long> geompos(0);
/* initial tile is 0/0/0 */ /* initial tile is normally 0/0/0 but can be iz/ix/iy if limited to one tile */
serialize_int(geomfile, iz, &geompos, fname); serialize_int(geomfile, iz, &geompos, fname);
serialize_uint(geomfile, ix, &geompos, fname); serialize_uint(geomfile, ix, &geompos, fname);
serialize_uint(geomfile, iy, &geompos, fname); serialize_uint(geomfile, iy, &geompos, fname);
@@ -2483,7 +2483,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
std::atomic<unsigned> midx(0); std::atomic<unsigned> midx(0);
std::atomic<unsigned> midy(0); std::atomic<unsigned> midy(0);
std::vector<strategy> strategies; std::vector<strategy> strategies;
int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies); int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz);
if (maxzoom != written) { if (maxzoom != written) {
if (written > minzoom) { if (written > minzoom) {
tile.cpp
+17 -11
View File
@@ -1643,6 +1643,7 @@ struct write_tile_args {
std::atomic<size_t> *dropped_count = NULL; std::atomic<size_t> *dropped_count = NULL;
atomic_strategy *strategy = NULL; atomic_strategy *strategy = NULL;
int zoom = -1; int zoom = -1;
bool compressed;
}; };
bool clip_to_tile(serial_feature &sf, int z, long long buffer) { bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
@@ -1740,7 +1741,7 @@ void remove_attributes(serial_feature &sf, std::set<std::string> const &exclude_
} }
} }
serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, compressor **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time) { serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, compressor **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed) {
while (1) { while (1) {
serial_feature sf; serial_feature sf;
std::string s; std::string s;
@@ -1748,7 +1749,7 @@ serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos
if (geoms->deserialize_long_long(&len, geompos_in) == 0) { if (geoms->deserialize_long_long(&len, geompos_in) == 0) {
printf("------ <<<<<< deserialization failure, end of tile\n"); printf("------ <<<<<< deserialization failure, end of tile\n");
if (z != 0) { if (compressed) {
geoms->end(geompos_in); geoms->end(geompos_in);
} }
@@ -1757,7 +1758,7 @@ serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos
} }
if (len == 0) { if (len == 0) {
printf("------ <<<<<< feature len 0, end of tile\n"); printf("------ <<<<<< feature len 0, end of tile\n");
if (z != 0) { if (compressed) {
geoms->end(geompos_in); geoms->end(geompos_in);
} }
@@ -1930,6 +1931,7 @@ struct run_prefilter_args {
FILE *prefilter_fp = NULL; FILE *prefilter_fp = NULL;
struct json_object *filter = NULL; struct json_object *filter = NULL;
bool first_time = false; bool first_time = false;
bool compressed = false;
}; };
void *run_prefilter(void *v) { void *run_prefilter(void *v) {
@@ -1937,7 +1939,7 @@ void *run_prefilter(void *v) {
json_writer state(rpa->prefilter_fp); json_writer state(rpa->prefilter_fp);
while (1) { while (1) {
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time); serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed);
if (sf.t < 0) { if (sf.t < 0) {
break; break;
} }
@@ -2187,7 +2189,7 @@ void add_sample_to(std::vector<T> &vals, T val, size_t &increment, size_t seq) {
} }
} }
long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) { long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy, bool compressed_input) {
double merge_fraction = 1; double merge_fraction = 1;
double mingap_fraction = 1; double mingap_fraction = 1;
double minextent_fraction = 1; double minextent_fraction = 1;
@@ -2259,7 +2261,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
} }
if (*geompos_in != og) { if (*geompos_in != og) {
if (z != 0) { if (compressed_input) {
long long n = *geompos_in; long long n = *geompos_in;
printf("-------- seek: end %lld to %lld\n", n, og); printf("-------- seek: end %lld to %lld\n", n, og);
geoms->end(geompos_in); geoms->end(geompos_in);
@@ -2326,6 +2328,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
rpa.pool_off = pool_off; rpa.pool_off = pool_off;
rpa.filter = filter; rpa.filter = filter;
rpa.first_time = first_time; rpa.first_time = first_time;
rpa.compressed = compressed_input;
if (pthread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) { if (pthread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) {
perror("pthread_create (prefilter writer)"); perror("pthread_create (prefilter writer)");
@@ -2345,7 +2348,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
ssize_t which_partial = -1; ssize_t which_partial = -1;
if (prefilter == NULL) { if (prefilter == NULL) {
sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time); sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time, compressed_input);
} else { } else {
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
} }
@@ -3193,12 +3196,14 @@ void *run_thread(void *vargs) {
dc.deserialize_uint(&x, &geompos); dc.deserialize_uint(&x, &geompos);
dc.deserialize_uint(&y, &geompos); dc.deserialize_uint(&y, &geompos);
printf("------- %d/%d/%d\n", z, x, y); printf("------- %d/%d/%d\n", z, x, y);
#if 0
if (z != arg->zoom) { if (z != arg->zoom) {
fprintf(stderr, "Expected zoom %d, found zoom %d\n", arg->zoom, z); fprintf(stderr, "Expected zoom %d, found zoom %d\n", arg->zoom, z);
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
#endif
if (z != 0) { if (arg->compressed) {
dc.begin(); dc.begin();
} }
@@ -3206,7 +3211,7 @@ void *run_thread(void *vargs) {
// fprintf(stderr, "%d/%u/%u\n", z, x, y); // fprintf(stderr, "%d/%u/%u\n", z, x, y);
long long len = write_tile(&dc, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy); long long len = write_tile(&dc, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed);
if (len < 0) { if (len < 0) {
int *err = &arg->err; int *err = &arg->err;
@@ -3271,7 +3276,7 @@ void *run_thread(void *vargs) {
return NULL; return NULL;
} }
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies) { int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz) {
last_progress = 0; last_progress = 0;
// The existing layermaps are one table per input thread. // The existing layermaps are one table per input thread.
@@ -3296,7 +3301,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<
} }
int z; int z;
for (z = 0; z <= maxzoom; z++) { for (z = iz; z <= maxzoom; z++) {
std::atomic<long long> most(0); std::atomic<long long> most(0);
compressor compressors[TEMP_FILES]; compressor compressors[TEMP_FILES];
@@ -3475,6 +3480,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<
args[thread].still_dropping = false; args[thread].still_dropping = false;
args[thread].strategy = &strategy; args[thread].strategy = &strategy;
args[thread].zoom = z; args[thread].zoom = z;
args[thread].compressed = (z != iz);
if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) { if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) {
perror("pthread_create"); perror("pthread_create");
tile.hpp
+1 -1
View File
@@ -63,7 +63,7 @@ struct strategy {
long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy); long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies); int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);