From d05f8f8e4cdc965d06bb5fb2c6222817f89c231a Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Thu, 29 Dec 2022 14:44:15 -0800 Subject: [PATCH] Remove the concept of "separate metadata" This was an extra level of attribute indirection (features point to metadata records which point to key and value strings) which was intended to reduce the size of temporary storage for features with large numbers of attributes that were also spread across large numbers of tiles at maxzoom. For other kinds of features, the extra indirection slowed things down instead, and, especially when maxzoom guessing was being used, many more features were having their metadata externalized than could actually benefit from it. --- main.cpp | 101 ++++------------------------------------------------- plugin.cpp | 1 - serial.cpp | 89 +++++++++------------------------------------- serial.hpp | 18 ++-------- tile.cpp | 35 +++++++------------ tile.hpp | 4 +-- 6 files changed, 40 insertions(+), 208 deletions(-) diff --git a/main.cpp b/main.cpp index 4c5ab3aa..2230b16a 100644 --- a/main.cpp +++ b/main.cpp @@ -113,9 +113,9 @@ std::vector clipbboxes; void checkdisk(std::vector *r) { long long used = 0; for (size_t i = 0; i < r->size(); i++) { - // Meta, pool, and tree are used once. + // Pool and tree are used once. // Geometry and index will be duplicated during sorting and tiling. - used += (*r)[i].metapos + 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len; + used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len; } static int warned = 0; @@ -1066,7 +1066,7 @@ void radix(std::vector &readers, int nreaders, FILE *geomfile, FI } long long availfiles = MAX_FILES - 2 * nreaders // each reader has a geom and an index - - 4 // pool, meta, mbtiles, mbtiles journal + - 3 // pool, mbtiles, mbtiles journal - 4 // top-level geom and index output, both FILE and fd - 3; // stdin, stdout, stderr @@ -1164,23 +1164,16 @@ std::pair read_input(std::vector &sources, char *fname, i for (size_t i = 0; i < CPUS; i++) { struct reader *r = &readers[i]; - char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; char treename[strlen(tmpdir) + strlen("/tree.XXXXXXXX") + 1]; char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1]; char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1]; - sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX"); sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX"); sprintf(treename, "%s%s", tmpdir, "/tree.XXXXXXXX"); sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX"); sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX"); - r->metafd = mkstemp_cloexec(metaname); - if (r->metafd < 0) { - perror(metaname); - exit(EXIT_OPEN); - } r->poolfd = mkstemp_cloexec(poolname); if (r->poolfd < 0) { perror(poolname); @@ -1202,11 +1195,6 @@ std::pair read_input(std::vector &sources, char *fname, i exit(EXIT_OPEN); } - r->metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC); - if (r->metafile == NULL) { - perror(metaname); - exit(EXIT_OPEN); - } r->poolfile = memfile_open(r->poolfd); if (r->poolfile == NULL) { perror(poolname); @@ -1227,11 +1215,9 @@ std::pair read_input(std::vector &sources, char *fname, i perror(indexname); exit(EXIT_OPEN); } - r->metapos = 0; r->geompos = 0; r->indexpos = 0; - unlink(metaname); unlink(poolname); unlink(treename); unlink(geomname); @@ -1242,8 +1228,6 @@ std::pair read_input(std::vector &sources, char *fname, i struct stringpool p; memfile_write(r->treefile, &p, sizeof(struct stringpool)); } - // Keep metadata file from being completely empty if no attributes - serialize_int(r->metafile, 0, &r->metapos, "meta"); r->file_bbox[0] = r->file_bbox[1] = UINT_MAX; r->file_bbox[2] = r->file_bbox[3] = 0; @@ -1806,10 +1790,6 @@ std::pair read_input(std::vector &sources, char *fname, i } for (size_t i = 0; i < CPUS; i++) { - if (fclose(readers[i].metafile) != 0) { - perror("fclose meta"); - exit(EXIT_CLOSE); - } if (fclose(readers[i].geomfile) != 0) { perror("fclose geom"); exit(EXIT_CLOSE); @@ -1824,21 +1804,16 @@ std::pair read_input(std::vector &sources, char *fname, i perror("stat geom\n"); exit(EXIT_STAT); } - if (fstat(readers[i].metafd, &readers[i].metast) != 0) { - perror("stat meta\n"); - exit(EXIT_STAT); - } } - // Create a combined string pool and a combined metadata file + // Create a combined string pool // but keep track of the offsets into it since we still need // segment+offset to find the data. // 2 * CPUS: One per input thread, one per tiling thread long long pool_off[2 * CPUS]; - long long meta_off[2 * CPUS]; for (size_t i = 0; i < 2 * CPUS; i++) { - pool_off[i] = meta_off[i] = 0; + pool_off[i] = 0; } char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1]; @@ -1858,51 +1833,9 @@ std::pair read_input(std::vector &sources, char *fname, i unlink(poolname); - char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1]; - sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX"); - - int metafd = mkstemp_cloexec(metaname); - if (metafd < 0) { - perror(metaname); - exit(EXIT_OPEN); - } - - FILE *metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC); - if (metafile == NULL) { - perror(metaname); - exit(EXIT_OPEN); - } - - unlink(metaname); - - std::atomic metapos(0); std::atomic poolpos(0); for (size_t i = 0; i < CPUS; i++) { - if (readers[i].metapos > 0) { - void *map = mmap(NULL, readers[i].metapos, PROT_READ, MAP_PRIVATE, readers[i].metafd, 0); - if (map == MAP_FAILED) { - perror("mmap unmerged meta"); - exit(EXIT_MEMORY); - } - madvise(map, readers[i].metapos, MADV_SEQUENTIAL); - madvise(map, readers[i].metapos, MADV_WILLNEED); - if (fwrite(map, readers[i].metapos, 1, metafile) != 1) { - perror("Reunify meta"); - exit(EXIT_WRITE); - } - madvise(map, readers[i].metapos, MADV_DONTNEED); - if (munmap(map, readers[i].metapos) != 0) { - perror("unmap unmerged meta"); - } - } - - meta_off[i] = metapos; - metapos += readers[i].metapos; - if (close(readers[i].metafd) != 0) { - perror("close unmerged meta"); - } - if (readers[i].poolfile->off > 0) { if (fwrite(readers[i].poolfile->map, readers[i].poolfile->off, 1, poolfile) != 1) { perror("Reunify string pool"); @@ -1919,17 +1852,6 @@ std::pair read_input(std::vector &sources, char *fname, i perror("fclose pool"); exit(EXIT_CLOSE); } - if (fclose(metafile) != 0) { - perror("fclose meta"); - exit(EXIT_CLOSE); - } - - char *meta = (char *) mmap(NULL, metapos, PROT_READ, MAP_PRIVATE, metafd, 0); - if (meta == MAP_FAILED) { - perror("mmap meta"); - exit(EXIT_MEMORY); - } - madvise(meta, metapos, MADV_RANDOM); char *stringpool = NULL; if (poolpos > 0) { // Will be 0 if -X was specified @@ -2014,9 +1936,8 @@ std::pair read_input(std::vector &sources, char *fname, i if (!quiet) { long long s = progress_seq; long long geompos_print = geompos; - long long metapos_print = metapos; long long poolpos_print = poolpos; - fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", s, geompos_print, metapos_print, poolpos_print); + fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of string pool\n", s, geompos_print, poolpos_print); } if (indexpos == 0) { @@ -2504,7 +2425,7 @@ std::pair read_input(std::vector &sources, char *fname, i std::atomic midx(0); std::atomic midy(0); std::vector strategies; - int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies); + int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies); if (maxzoom != written) { if (written > minzoom) { @@ -2517,14 +2438,6 @@ std::pair read_input(std::vector &sources, char *fname, i } } - madvise(meta, metapos, MADV_DONTNEED); - if (munmap(meta, metapos) != 0) { - perror("munmap meta"); - } - if (close(metafd) < 0) { - perror("close meta"); - } - if (poolpos > 0) { madvise((void *) stringpool, poolpos, MADV_DONTNEED); if (munmap(stringpool, poolpos) != 0) { diff --git a/plugin.cpp b/plugin.cpp index 158ea3f7..0e65509f 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -405,7 +405,6 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: sf.bbox[0] = sf.bbox[1] = LLONG_MAX; sf.bbox[2] = sf.bbox[3] = LLONG_MIN; sf.extent = 0; - sf.metapos = 0; sf.has_id = false; std::string layername = "unknown"; diff --git a/serial.cpp b/serial.cpp index 00e4366e..c9392a34 100644 --- a/serial.cpp +++ b/serial.cpp @@ -240,15 +240,11 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomicextent, geompos, fname); } - serialize_long_long(geomfile, sf->metapos, geompos, fname); + serialize_long_long(geomfile, sf->keys.size(), geompos, fname); - if (sf->metapos < 0) { - serialize_long_long(geomfile, sf->keys.size(), geompos, fname); - - for (size_t i = 0; i < sf->keys.size(); i++) { - serialize_long_long(geomfile, sf->keys[i], geompos, fname); - serialize_long_long(geomfile, sf->values[i], geompos, fname); - } + for (size_t i = 0; i < sf->keys.size(); i++) { + serialize_long_long(geomfile, sf->keys[i], geompos, fname); + serialize_long_long(geomfile, sf->values[i], geompos, fname); } if (include_minzoom) { @@ -256,7 +252,7 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) { +serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) { serial_feature sf; deserialize_byte_io(geoms, &sf.t, geompos_in); @@ -305,32 +301,15 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_ sf.layer >>= FLAG_LAYER; - sf.metapos = 0; - deserialize_long_long_io(geoms, &sf.metapos, geompos_in); + long long count; + deserialize_long_long_io(geoms, &count, geompos_in); - if (sf.metapos >= 0) { - char *meta = metabase + sf.metapos + meta_off[sf.segment]; - long long count; - deserialize_long_long(&meta, &count); - - for (long long i = 0; i < count; i++) { - long long k, v; - deserialize_long_long(&meta, &k); - deserialize_long_long(&meta, &v); - sf.keys.push_back(k); - sf.values.push_back(v); - } - } else { - long long count; - deserialize_long_long_io(geoms, &count, geompos_in); - - for (long long i = 0; i < count; i++) { - long long k, v; - deserialize_long_long_io(geoms, &k, geompos_in); - deserialize_long_long_io(geoms, &v, geompos_in); - sf.keys.push_back(k); - sf.values.push_back(v); - } + for (long long i = 0; i < count; i++) { + long long k, v; + deserialize_long_long_io(geoms, &k, geompos_in); + deserialize_long_long_io(geoms, &v, geompos_in); + sf.keys.push_back(k); + sf.values.push_back(v); } deserialize_byte_io(geoms, &sf.feature_minzoom, geompos_in); @@ -512,32 +491,6 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { locs.clear(); } - bool inline_meta = true; - // Don't inline metadata for features that will span several tiles at maxzoom - if (scaled_geometry.size() > 0 && (sf.bbox[2] < sf.bbox[0] || sf.bbox[3] < sf.bbox[1])) { - fprintf(stderr, "Internal error: impossible feature bounding box %llx,%llx,%llx,%llx\n", sf.bbox[0], sf.bbox[1], sf.bbox[2], sf.bbox[3]); - } - if (sf.bbox[0] == LLONG_MAX) { - // No bounding box (empty geometry) - // Shouldn't happen, but avoid arithmetic overflow below - } else if (sf.bbox[2] - sf.bbox[0] > (2LL << (32 - sst->maxzoom)) || sf.bbox[3] - sf.bbox[1] > (2LL << (32 - sst->maxzoom))) { - inline_meta = false; - - if (prevent[P_CLIPPING]) { - static std::atomic warned(0); - long long extent = ((sf.bbox[2] - sf.bbox[0]) / ((1LL << (32 - sst->maxzoom)) + 1)) * ((sf.bbox[3] - sf.bbox[1]) / ((1LL << (32 - sst->maxzoom)) + 1)); - if (extent > warned) { - fprintf(stderr, "Warning: %s:%d: Large unclipped (-pc) feature may be duplicated across %lld tiles\n", sst->fname, sst->line, extent); - warned = extent; - - if (extent > 10000) { - fprintf(stderr, "Exiting because this can't be right.\n"); - exit(EXIT_IMPOSSIBLE); - } - } - } - } - double extent = 0; if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED] || order_by_size || sst->want_dist) { if (sf.t == VT_POLYGON) { @@ -725,19 +678,9 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { } } - if (inline_meta) { - sf.metapos = -1; - for (size_t i = 0; i < sf.full_keys.size(); i++) { - sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string)); - sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type)); - } - } else { - sf.metapos = r->metapos; - serialize_long_long(r->metafile, sf.full_keys.size(), &r->metapos, sst->fname); - for (size_t i = 0; i < sf.full_keys.size(); i++) { - serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string), &r->metapos, sst->fname); - serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type), &r->metapos, sst->fname); - } + for (size_t i = 0; i < sf.full_keys.size(); i++) { + sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string)); + sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type)); } long long geomstart = r->geompos; diff --git a/serial.hpp b/serial.hpp index 4ee92285..711d3fb5 100644 --- a/serial.hpp +++ b/serial.hpp @@ -61,8 +61,6 @@ struct serial_feature { std::vector keys{}; std::vector values{}; - // If >= 0, metadata is external - long long metapos = 0; // XXX This isn't serialized. Should it be here? long long bbox[4] = {0, 0, 0, 0}; @@ -73,53 +71,44 @@ struct serial_feature { }; void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic *geompos, const char *fname, long long wx, long long wy, bool include_minzoom); -serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y); +serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y); struct reader { - int metafd = -1; int poolfd = -1; int treefd = -1; int geomfd = -1; int indexfd = -1; - FILE *metafile = NULL; struct memfile *poolfile = NULL; struct memfile *treefile = NULL; FILE *geomfile = NULL; FILE *indexfile = NULL; - std::atomic metapos; std::atomic geompos; std::atomic indexpos; long long file_bbox[4] = {0, 0, 0, 0}; struct stat geomst {}; - struct stat metast {}; char *geom_map = NULL; reader() - : metapos(0), geompos(0), indexpos(0) { + : geompos(0), indexpos(0) { } reader(reader const &r) { - metafd = r.metafd; poolfd = r.poolfd; treefd = r.treefd; geomfd = r.geomfd; indexfd = r.indexfd; - metafile = r.metafile; poolfile = r.poolfile; treefile = r.treefile; geomfile = r.geomfile; indexfile = r.indexfile; - long long p = r.metapos; - metapos = p; - - p = r.geompos; + long long p = r.geompos; geompos = p; p = r.indexpos; @@ -128,7 +117,6 @@ struct reader { memcpy(file_bbox, r.file_bbox, sizeof(file_bbox)); geomst = r.geomst; - metast = r.metast; geom_map = r.geom_map; } diff --git a/tile.cpp b/tile.cpp index f1a5db62..a252217a 100644 --- a/tile.cpp +++ b/tile.cpp @@ -324,7 +324,7 @@ struct ordercmp { } } ordercmp; -void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector &metakeys, std::vector &metavals, bool has_id, unsigned long long id, unsigned long long index, unsigned long long label_point, long long extent) { +void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic *geompos, FILE **geomfile, const char *fname, signed char t, int layer, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector &metakeys, std::vector &metavals, bool has_id, unsigned long long id, unsigned long long index, unsigned long long label_point, long long extent) { if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) { int xo, yo; int span = 1 << (nextzoom - z); @@ -411,18 +411,15 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u sf.tippecanoe_minzoom = tippecanoe_minzoom; sf.has_tippecanoe_maxzoom = tippecanoe_maxzoom != -1; sf.tippecanoe_maxzoom = tippecanoe_maxzoom; - sf.metapos = metastart; sf.geometry = geom2; sf.index = index; sf.label_point = label_point; sf.extent = extent; sf.feature_minzoom = feature_minzoom; - if (metastart < 0) { - for (size_t i = 0; i < metakeys.size(); i++) { - sf.keys.push_back(metakeys[i]); - sf.values.push_back(metavals[i]); - } + for (size_t i = 0; i < metakeys.size(); i++) { + sf.keys.push_back(metakeys[i]); + sf.values.push_back(metavals[i]); } serialize_feature(geomfile[j], &sf, &geompos[j], fname, SHIFT_RIGHT(initial_x[segment]), SHIFT_RIGHT(initial_y[segment]), true); @@ -1283,7 +1280,6 @@ long long choose_minextent(std::vector &extents, double f) { struct write_tile_args { struct task *tasks = NULL; - char *metabase = NULL; char *stringpool = NULL; int min_detail = 0; sqlite3 *outdb = NULL; @@ -1306,7 +1302,6 @@ struct write_tile_args { int low_detail = 0; double simplification = 0; std::atomic *most = NULL; - long long *meta_off = NULL; long long *pool_off = NULL; unsigned *initial_x = NULL; unsigned *initial_y = NULL; @@ -1429,9 +1424,9 @@ void remove_attributes(serial_feature &sf, std::set const &exclude_ } } -serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, int *within, FILE **geomfile, std::atomic *geompos, std::atomic *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time) { +serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, int *within, FILE **geomfile, std::atomic *geompos, std::atomic *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time) { while (1) { - serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y); + serial_feature sf = deserialize_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y); if (sf.t < 0) { return sf; } @@ -1460,7 +1455,7 @@ serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, cha if (first_time && pass == 0) { /* only write out the next zoom once, even if we retry */ if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) { - rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.label_point, sf.extent); + rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.label_point, sf.extent); } } @@ -1563,8 +1558,6 @@ serial_feature next_feature(FILE *geoms, std::atomic *geompos_in, cha struct run_prefilter_args { FILE *geoms = NULL; std::atomic *geompos_in = NULL; - char *metabase = NULL; - long long *meta_off = NULL; int z = 0; unsigned tx = 0; unsigned ty = 0; @@ -1600,7 +1593,7 @@ void *run_prefilter(void *v) { json_writer state(rpa->prefilter_fp); while (1) { - serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time); + serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time); if (sf.t < 0) { break; } @@ -1850,7 +1843,7 @@ void add_sample_to(std::vector &vals, T val, size_t &increment, size_t seq) { } } -long long write_tile(FILE *geoms, std::atomic *geompos_in, char *metabase, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) { +long long write_tile(FILE *geoms, std::atomic *geompos_in, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) { double merge_fraction = 1; double mingap_fraction = 1; double minextent_fraction = 1; @@ -1954,8 +1947,6 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta rpa.geoms = geoms; rpa.geompos_in = geompos_in; - rpa.metabase = metabase; - rpa.meta_off = meta_off; rpa.z = z; rpa.tx = tx; rpa.ty = ty; @@ -2003,7 +1994,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *meta ssize_t which_partial = -1; if (prefilter == NULL) { - sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time); + sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time); } else { sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); } @@ -2837,7 +2828,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy); + long long len = write_tile(geom, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy); if (len < 0) { int *err = &arg->err; @@ -2902,7 +2893,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies) { last_progress = 0; // The existing layermaps are one table per input thread. @@ -3049,7 +3040,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo atomic_strategy strategy; for (size_t thread = 0; thread < threads; thread++) { - args[thread].metabase = metabase; args[thread].stringpool = stringpool; args[thread].min_detail = min_detail; args[thread].outdb = outdb; // locked with db_lock @@ -3086,7 +3076,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].full_detail = full_detail; args[thread].low_detail = low_detail; args[thread].most = &most; // locked with var_lock - args[thread].meta_off = meta_off; args[thread].pool_off = pool_off; args[thread].initial_x = initial_x; args[thread].initial_y = initial_y; diff --git a/tile.hpp b/tile.hpp index 913ae81d..2b77d80a 100644 --- a/tile.hpp +++ b/tile.hpp @@ -61,9 +61,9 @@ struct strategy { strategy() = default; }; -long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); +long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);