diff --git a/main.cpp b/main.cpp index 8a021001..4df86043 100644 --- a/main.cpp +++ b/main.cpp @@ -1947,7 +1947,7 @@ std::pair read_input(std::vector &sources, char *fname, i std::atomic geompos(0); - /* initial tile is 0/0/0 */ + /* initial tile is normally 0/0/0 but can be iz/ix/iy if limited to one tile */ serialize_int(geomfile, iz, &geompos, fname); serialize_uint(geomfile, ix, &geompos, fname); serialize_uint(geomfile, iy, &geompos, fname); @@ -2483,7 +2483,7 @@ std::pair read_input(std::vector &sources, char *fname, i std::atomic midx(0); std::atomic midy(0); std::vector strategies; - int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies); + int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz); if (maxzoom != written) { if (written > minzoom) { diff --git a/tile.cpp b/tile.cpp index b4d33db4..34bbbc03 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1643,6 +1643,7 @@ struct write_tile_args { std::atomic *dropped_count = NULL; atomic_strategy *strategy = NULL; int zoom = -1; + bool compressed; }; bool clip_to_tile(serial_feature &sf, int z, long long buffer) { @@ -1740,7 +1741,7 @@ void remove_attributes(serial_feature &sf, std::set const &exclude_ } } -serial_feature next_feature(decompressor *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, int *within, compressor **geomfile, std::atomic *geompos, std::atomic *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time) { +serial_feature next_feature(decompressor *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, int *within, compressor **geomfile, std::atomic *geompos, std::atomic *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time, bool compressed) { while (1) { serial_feature sf; std::string s; @@ -1748,7 +1749,7 @@ serial_feature next_feature(decompressor *geoms, std::atomic *geompos if (geoms->deserialize_long_long(&len, geompos_in) == 0) { printf("------ <<<<<< deserialization failure, end of tile\n"); - if (z != 0) { + if (compressed) { geoms->end(geompos_in); } @@ -1757,7 +1758,7 @@ serial_feature next_feature(decompressor *geoms, std::atomic *geompos } if (len == 0) { printf("------ <<<<<< feature len 0, end of tile\n"); - if (z != 0) { + if (compressed) { geoms->end(geompos_in); } @@ -1930,6 +1931,7 @@ struct run_prefilter_args { FILE *prefilter_fp = NULL; struct json_object *filter = NULL; bool first_time = false; + bool compressed = false; }; void *run_prefilter(void *v) { @@ -1937,7 +1939,7 @@ void *run_prefilter(void *v) { json_writer state(rpa->prefilter_fp); while (1) { - serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time); + serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed); if (sf.t < 0) { break; } @@ -2187,7 +2189,7 @@ void add_sample_to(std::vector &vals, T val, size_t &increment, size_t seq) { } } -long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) { +long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy, bool compressed_input) { double merge_fraction = 1; double mingap_fraction = 1; double minextent_fraction = 1; @@ -2259,7 +2261,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } if (*geompos_in != og) { - if (z != 0) { + if (compressed_input) { long long n = *geompos_in; printf("-------- seek: end %lld to %lld\n", n, og); geoms->end(geompos_in); @@ -2326,6 +2328,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch rpa.pool_off = pool_off; rpa.filter = filter; rpa.first_time = first_time; + rpa.compressed = compressed_input; if (pthread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) { perror("pthread_create (prefilter writer)"); @@ -2345,7 +2348,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch ssize_t which_partial = -1; if (prefilter == NULL) { - sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time); + sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time, compressed_input); } else { sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); } @@ -3193,12 +3196,14 @@ void *run_thread(void *vargs) { dc.deserialize_uint(&x, &geompos); dc.deserialize_uint(&y, &geompos); printf("------- %d/%d/%d\n", z, x, y); +#if 0 if (z != arg->zoom) { fprintf(stderr, "Expected zoom %d, found zoom %d\n", arg->zoom, z); exit(EXIT_IMPOSSIBLE); } +#endif - if (z != 0) { + if (arg->compressed) { dc.begin(); } @@ -3206,7 +3211,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(&dc, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy); + long long len = write_tile(&dc, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed); if (len < 0) { int *err = &arg->err; @@ -3271,7 +3276,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz) { last_progress = 0; // The existing layermaps are one table per input thread. @@ -3296,7 +3301,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic< } int z; - for (z = 0; z <= maxzoom; z++) { + for (z = iz; z <= maxzoom; z++) { std::atomic most(0); compressor compressors[TEMP_FILES]; @@ -3475,6 +3480,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic< args[thread].still_dropping = false; args[thread].strategy = &strategy; args[thread].zoom = z; + args[thread].compressed = (z != iz); if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) { perror("pthread_create"); diff --git a/tile.hpp b/tile.hpp index 2b77d80a..1159f376 100644 --- a/tile.hpp +++ b/tile.hpp @@ -63,7 +63,7 @@ struct strategy { long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);