From 80cd39d99600208e6edaa10d2282fca40bd1c55f Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Thu, 9 Feb 2023 13:06:44 -0800 Subject: [PATCH] Pass the merged layermaps into tiling --- main.cpp | 8 ++++++-- tile.cpp | 8 +++++--- tile.hpp | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/main.cpp b/main.cpp index 0f7b3cfa..febeca2c 100644 --- a/main.cpp +++ b/main.cpp @@ -2496,6 +2496,9 @@ std::pair read_input(std::vector &sources, char *fname, i perror("close sorted index"); } + // provisional merged tilestats, so the stats for each attribute can be used during tiling + std::map merged_lm = merge_layermaps(layermaps); + /* Traverse and split the geometries for each zoom level */ struct stat geomst; @@ -2518,7 +2521,7 @@ std::pair read_input(std::vector &sources, char *fname, i std::atomic midx(0); std::atomic midy(0); std::vector strategies; - int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies); + int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, merged_lm, prefilter, postfilter, attribute_accum, filter, strategies); if (maxzoom != written) { if (written > minzoom) { @@ -2573,7 +2576,8 @@ std::pair read_input(std::vector &sources, char *fname, i midlon = maxlon; } - std::map merged_lm = merge_layermaps(layermaps); + // merge again, because new entries can have been added during tiling + merged_lm = merge_layermaps(layermaps); for (auto ai = merged_lm.begin(); ai != merged_lm.end(); ++ai) { ai->second.minzoom = minzoom; diff --git a/tile.cpp b/tile.cpp index fd6fbeeb..6e4e095a 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1434,6 +1434,7 @@ struct write_tile_args { int err = 0; std::vector> *layermaps = NULL; std::vector> *layer_unmaps = NULL; + const std::map *merged_layermaps = NULL; size_t pass = 0; unsigned long long mingap = 0; unsigned long long mingap_out = 0; @@ -1970,7 +1971,7 @@ void add_sample_to(std::vector &vals, T val, size_t &increment, size_t seq) { } } -long long write_tile(FILE *geoms, std::atomic *geompos_in, char *metabase, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) { +long long write_tile(FILE *geoms, std::atomic *geompos_in, char *metabase, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, const std::map *merged_layermaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) { double merge_fraction = 1; double mingap_fraction = 1; double minextent_fraction = 1; @@ -2941,7 +2942,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy); + long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->merged_layermaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy); if (len < 0) { int *err = &arg->err; @@ -3006,7 +3007,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const std::map &merged_layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies) { last_progress = 0; // The existing layermaps are one table per input thread. @@ -3196,6 +3197,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].initial_y = initial_y; args[thread].layermaps = &layermaps; args[thread].layer_unmaps = &layer_unmaps; + args[thread].merged_layermaps = &merged_layermaps; args[thread].tiling_seg = thread + layermaps_off; args[thread].prefilter = prefilter; args[thread].postfilter = postfilter; diff --git a/tile.hpp b/tile.hpp index 913ae81d..9fcdc9b8 100644 --- a/tile.hpp +++ b/tile.hpp @@ -63,7 +63,7 @@ struct strategy { long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const std::map &merged_layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);