mirror of
https://github.com/felt/tippecanoe.git
synced 2026-10-02 08:25:40 +02:00
Remove the concept of "separate metadata"
This was an extra level of attribute indirection (features point to metadata records which point to key and value strings) which was intended to reduce the size of temporary storage for features with large numbers of attributes that were also spread across large numbers of tiles at maxzoom. For other kinds of features, the extra indirection slowed things down instead, and, especially when maxzoom guessing was being used, many more features were having their metadata externalized than could actually benefit from it.
This commit is contained in:
Erica Fischer
@@ -113,9 +113,9 @@ std::vector<clipbbox> clipbboxes;
|
||||
void checkdisk(std::vector<struct reader> *r) {
|
||||
long long used = 0;
|
||||
for (size_t i = 0; i < r->size(); i++) {
|
||||
// Meta, pool, and tree are used once.
|
||||
// Pool and tree are used once.
|
||||
// Geometry and index will be duplicated during sorting and tiling.
|
||||
used += (*r)[i].metapos + 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len;
|
||||
used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len;
|
||||
}
|
||||
|
||||
static int warned = 0;
|
||||
@@ -1066,7 +1066,7 @@ void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FI
|
||||
}
|
||||
|
||||
long long availfiles = MAX_FILES - 2 * nreaders // each reader has a geom and an index
|
||||
- 4 // pool, meta, mbtiles, mbtiles journal
|
||||
- 3 // pool, mbtiles, mbtiles journal
|
||||
- 4 // top-level geom and index output, both FILE and fd
|
||||
- 3; // stdin, stdout, stderr
|
||||
|
||||
@@ -1164,23 +1164,16 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
struct reader *r = &readers[i];
|
||||
|
||||
char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1];
|
||||
char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1];
|
||||
char treename[strlen(tmpdir) + strlen("/tree.XXXXXXXX") + 1];
|
||||
char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1];
|
||||
char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1];
|
||||
|
||||
sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX");
|
||||
sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX");
|
||||
sprintf(treename, "%s%s", tmpdir, "/tree.XXXXXXXX");
|
||||
sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX");
|
||||
sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX");
|
||||
|
||||
r->metafd = mkstemp_cloexec(metaname);
|
||||
if (r->metafd < 0) {
|
||||
perror(metaname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
r->poolfd = mkstemp_cloexec(poolname);
|
||||
if (r->poolfd < 0) {
|
||||
perror(poolname);
|
||||
@@ -1202,11 +1195,6 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
|
||||
r->metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC);
|
||||
if (r->metafile == NULL) {
|
||||
perror(metaname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
r->poolfile = memfile_open(r->poolfd);
|
||||
if (r->poolfile == NULL) {
|
||||
perror(poolname);
|
||||
@@ -1227,11 +1215,9 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
perror(indexname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
r->metapos = 0;
|
||||
r->geompos = 0;
|
||||
r->indexpos = 0;
|
||||
|
||||
unlink(metaname);
|
||||
unlink(poolname);
|
||||
unlink(treename);
|
||||
unlink(geomname);
|
||||
@@ -1242,8 +1228,6 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
struct stringpool p;
|
||||
memfile_write(r->treefile, &p, sizeof(struct stringpool));
|
||||
}
|
||||
// Keep metadata file from being completely empty if no attributes
|
||||
serialize_int(r->metafile, 0, &r->metapos, "meta");
|
||||
|
||||
r->file_bbox[0] = r->file_bbox[1] = UINT_MAX;
|
||||
r->file_bbox[2] = r->file_bbox[3] = 0;
|
||||
@@ -1806,10 +1790,6 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
if (fclose(readers[i].metafile) != 0) {
|
||||
perror("fclose meta");
|
||||
exit(EXIT_CLOSE);
|
||||
}
|
||||
if (fclose(readers[i].geomfile) != 0) {
|
||||
perror("fclose geom");
|
||||
exit(EXIT_CLOSE);
|
||||
@@ -1824,21 +1804,16 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
perror("stat geom\n");
|
||||
exit(EXIT_STAT);
|
||||
}
|
||||
if (fstat(readers[i].metafd, &readers[i].metast) != 0) {
|
||||
perror("stat meta\n");
|
||||
exit(EXIT_STAT);
|
||||
}
|
||||
}
|
||||
|
||||
// Create a combined string pool and a combined metadata file
|
||||
// Create a combined string pool
|
||||
// but keep track of the offsets into it since we still need
|
||||
// segment+offset to find the data.
|
||||
|
||||
// 2 * CPUS: One per input thread, one per tiling thread
|
||||
long long pool_off[2 * CPUS];
|
||||
long long meta_off[2 * CPUS];
|
||||
for (size_t i = 0; i < 2 * CPUS; i++) {
|
||||
pool_off[i] = meta_off[i] = 0;
|
||||
pool_off[i] = 0;
|
||||
}
|
||||
|
||||
char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1];
|
||||
@@ -1858,51 +1833,9 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
|
||||
unlink(poolname);
|
||||
|
||||
char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1];
|
||||
sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX");
|
||||
|
||||
int metafd = mkstemp_cloexec(metaname);
|
||||
if (metafd < 0) {
|
||||
perror(metaname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
|
||||
FILE *metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC);
|
||||
if (metafile == NULL) {
|
||||
perror(metaname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
|
||||
unlink(metaname);
|
||||
|
||||
std::atomic<long long> metapos(0);
|
||||
std::atomic<long long> poolpos(0);
|
||||
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
if (readers[i].metapos > 0) {
|
||||
void *map = mmap(NULL, readers[i].metapos, PROT_READ, MAP_PRIVATE, readers[i].metafd, 0);
|
||||
if (map == MAP_FAILED) {
|
||||
perror("mmap unmerged meta");
|
||||
exit(EXIT_MEMORY);
|
||||
}
|
||||
madvise(map, readers[i].metapos, MADV_SEQUENTIAL);
|
||||
madvise(map, readers[i].metapos, MADV_WILLNEED);
|
||||
if (fwrite(map, readers[i].metapos, 1, metafile) != 1) {
|
||||
perror("Reunify meta");
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
madvise(map, readers[i].metapos, MADV_DONTNEED);
|
||||
if (munmap(map, readers[i].metapos) != 0) {
|
||||
perror("unmap unmerged meta");
|
||||
}
|
||||
}
|
||||
|
||||
meta_off[i] = metapos;
|
||||
metapos += readers[i].metapos;
|
||||
if (close(readers[i].metafd) != 0) {
|
||||
perror("close unmerged meta");
|
||||
}
|
||||
|
||||
if (readers[i].poolfile->off > 0) {
|
||||
if (fwrite(readers[i].poolfile->map, readers[i].poolfile->off, 1, poolfile) != 1) {
|
||||
perror("Reunify string pool");
|
||||
@@ -1919,17 +1852,6 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
perror("fclose pool");
|
||||
exit(EXIT_CLOSE);
|
||||
}
|
||||
if (fclose(metafile) != 0) {
|
||||
perror("fclose meta");
|
||||
exit(EXIT_CLOSE);
|
||||
}
|
||||
|
||||
char *meta = (char *) mmap(NULL, metapos, PROT_READ, MAP_PRIVATE, metafd, 0);
|
||||
if (meta == MAP_FAILED) {
|
||||
perror("mmap meta");
|
||||
exit(EXIT_MEMORY);
|
||||
}
|
||||
madvise(meta, metapos, MADV_RANDOM);
|
||||
|
||||
char *stringpool = NULL;
|
||||
if (poolpos > 0) { // Will be 0 if -X was specified
|
||||
@@ -2014,9 +1936,8 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
if (!quiet) {
|
||||
long long s = progress_seq;
|
||||
long long geompos_print = geompos;
|
||||
long long metapos_print = metapos;
|
||||
long long poolpos_print = poolpos;
|
||||
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", s, geompos_print, metapos_print, poolpos_print);
|
||||
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of string pool\n", s, geompos_print, poolpos_print);
|
||||
}
|
||||
|
||||
if (indexpos == 0) {
|
||||
@@ -2504,7 +2425,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
std::atomic<unsigned> midx(0);
|
||||
std::atomic<unsigned> midy(0);
|
||||
std::vector<strategy> strategies;
|
||||
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies);
|
||||
int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies);
|
||||
|
||||
if (maxzoom != written) {
|
||||
if (written > minzoom) {
|
||||
@@ -2517,14 +2438,6 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
}
|
||||
}
|
||||
|
||||
madvise(meta, metapos, MADV_DONTNEED);
|
||||
if (munmap(meta, metapos) != 0) {
|
||||
perror("munmap meta");
|
||||
}
|
||||
if (close(metafd) < 0) {
|
||||
perror("close meta");
|
||||
}
|
||||
|
||||
if (poolpos > 0) {
|
||||
madvise((void *) stringpool, poolpos, MADV_DONTNEED);
|
||||
if (munmap(stringpool, poolpos) != 0) {
|
||||
|
||||
@@ -405,7 +405,6 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
|
||||
sf.bbox[0] = sf.bbox[1] = LLONG_MAX;
|
||||
sf.bbox[2] = sf.bbox[3] = LLONG_MIN;
|
||||
sf.extent = 0;
|
||||
sf.metapos = 0;
|
||||
sf.has_id = false;
|
||||
|
||||
std::string layername = "unknown";
|
||||
|
||||
+1
-58
@@ -240,23 +240,19 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long
|
||||
serialize_long_long(geomfile, sf->extent, geompos, fname);
|
||||
}
|
||||
|
||||
serialize_long_long(geomfile, sf->metapos, geompos, fname);
|
||||
|
||||
if (sf->metapos < 0) {
|
||||
serialize_long_long(geomfile, sf->keys.size(), geompos, fname);
|
||||
|
||||
for (size_t i = 0; i < sf->keys.size(); i++) {
|
||||
serialize_long_long(geomfile, sf->keys[i], geompos, fname);
|
||||
serialize_long_long(geomfile, sf->values[i], geompos, fname);
|
||||
}
|
||||
}
|
||||
|
||||
if (include_minzoom) {
|
||||
serialize_byte(geomfile, sf->feature_minzoom, geompos, fname);
|
||||
}
|
||||
}
|
||||
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
|
||||
serial_feature sf;
|
||||
|
||||
deserialize_byte_io(geoms, &sf.t, geompos_in);
|
||||
@@ -305,22 +301,6 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_
|
||||
|
||||
sf.layer >>= FLAG_LAYER;
|
||||
|
||||
sf.metapos = 0;
|
||||
deserialize_long_long_io(geoms, &sf.metapos, geompos_in);
|
||||
|
||||
if (sf.metapos >= 0) {
|
||||
char *meta = metabase + sf.metapos + meta_off[sf.segment];
|
||||
long long count;
|
||||
deserialize_long_long(&meta, &count);
|
||||
|
||||
for (long long i = 0; i < count; i++) {
|
||||
long long k, v;
|
||||
deserialize_long_long(&meta, &k);
|
||||
deserialize_long_long(&meta, &v);
|
||||
sf.keys.push_back(k);
|
||||
sf.values.push_back(v);
|
||||
}
|
||||
} else {
|
||||
long long count;
|
||||
deserialize_long_long_io(geoms, &count, geompos_in);
|
||||
|
||||
@@ -331,7 +311,6 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_
|
||||
sf.keys.push_back(k);
|
||||
sf.values.push_back(v);
|
||||
}
|
||||
}
|
||||
|
||||
deserialize_byte_io(geoms, &sf.feature_minzoom, geompos_in);
|
||||
|
||||
@@ -512,32 +491,6 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
locs.clear();
|
||||
}
|
||||
|
||||
bool inline_meta = true;
|
||||
// Don't inline metadata for features that will span several tiles at maxzoom
|
||||
if (scaled_geometry.size() > 0 && (sf.bbox[2] < sf.bbox[0] || sf.bbox[3] < sf.bbox[1])) {
|
||||
fprintf(stderr, "Internal error: impossible feature bounding box %llx,%llx,%llx,%llx\n", sf.bbox[0], sf.bbox[1], sf.bbox[2], sf.bbox[3]);
|
||||
}
|
||||
if (sf.bbox[0] == LLONG_MAX) {
|
||||
// No bounding box (empty geometry)
|
||||
// Shouldn't happen, but avoid arithmetic overflow below
|
||||
} else if (sf.bbox[2] - sf.bbox[0] > (2LL << (32 - sst->maxzoom)) || sf.bbox[3] - sf.bbox[1] > (2LL << (32 - sst->maxzoom))) {
|
||||
inline_meta = false;
|
||||
|
||||
if (prevent[P_CLIPPING]) {
|
||||
static std::atomic<long long> warned(0);
|
||||
long long extent = ((sf.bbox[2] - sf.bbox[0]) / ((1LL << (32 - sst->maxzoom)) + 1)) * ((sf.bbox[3] - sf.bbox[1]) / ((1LL << (32 - sst->maxzoom)) + 1));
|
||||
if (extent > warned) {
|
||||
fprintf(stderr, "Warning: %s:%d: Large unclipped (-pc) feature may be duplicated across %lld tiles\n", sst->fname, sst->line, extent);
|
||||
warned = extent;
|
||||
|
||||
if (extent > 10000) {
|
||||
fprintf(stderr, "Exiting because this can't be right.\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
double extent = 0;
|
||||
if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED] || order_by_size || sst->want_dist) {
|
||||
if (sf.t == VT_POLYGON) {
|
||||
@@ -725,20 +678,10 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
}
|
||||
}
|
||||
|
||||
if (inline_meta) {
|
||||
sf.metapos = -1;
|
||||
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
||||
sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string));
|
||||
sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type));
|
||||
}
|
||||
} else {
|
||||
sf.metapos = r->metapos;
|
||||
serialize_long_long(r->metafile, sf.full_keys.size(), &r->metapos, sst->fname);
|
||||
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
||||
serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string), &r->metapos, sst->fname);
|
||||
serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type), &r->metapos, sst->fname);
|
||||
}
|
||||
}
|
||||
|
||||
long long geomstart = r->geompos;
|
||||
sf.geometry = scaled_geometry;
|
||||
|
||||
+3
-15
@@ -61,8 +61,6 @@ struct serial_feature {
|
||||
|
||||
std::vector<long long> keys{};
|
||||
std::vector<long long> values{};
|
||||
// If >= 0, metadata is external
|
||||
long long metapos = 0;
|
||||
|
||||
// XXX This isn't serialized. Should it be here?
|
||||
long long bbox[4] = {0, 0, 0, 0};
|
||||
@@ -73,53 +71,44 @@ struct serial_feature {
|
||||
};
|
||||
|
||||
void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long> *geompos, const char *fname, long long wx, long long wy, bool include_minzoom);
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
|
||||
|
||||
struct reader {
|
||||
int metafd = -1;
|
||||
int poolfd = -1;
|
||||
int treefd = -1;
|
||||
int geomfd = -1;
|
||||
int indexfd = -1;
|
||||
|
||||
FILE *metafile = NULL;
|
||||
struct memfile *poolfile = NULL;
|
||||
struct memfile *treefile = NULL;
|
||||
FILE *geomfile = NULL;
|
||||
FILE *indexfile = NULL;
|
||||
|
||||
std::atomic<long long> metapos;
|
||||
std::atomic<long long> geompos;
|
||||
std::atomic<long long> indexpos;
|
||||
|
||||
long long file_bbox[4] = {0, 0, 0, 0};
|
||||
|
||||
struct stat geomst {};
|
||||
struct stat metast {};
|
||||
|
||||
char *geom_map = NULL;
|
||||
|
||||
reader()
|
||||
: metapos(0), geompos(0), indexpos(0) {
|
||||
: geompos(0), indexpos(0) {
|
||||
}
|
||||
|
||||
reader(reader const &r) {
|
||||
metafd = r.metafd;
|
||||
poolfd = r.poolfd;
|
||||
treefd = r.treefd;
|
||||
geomfd = r.geomfd;
|
||||
indexfd = r.indexfd;
|
||||
|
||||
metafile = r.metafile;
|
||||
poolfile = r.poolfile;
|
||||
treefile = r.treefile;
|
||||
geomfile = r.geomfile;
|
||||
indexfile = r.indexfile;
|
||||
|
||||
long long p = r.metapos;
|
||||
metapos = p;
|
||||
|
||||
p = r.geompos;
|
||||
long long p = r.geompos;
|
||||
geompos = p;
|
||||
|
||||
p = r.indexpos;
|
||||
@@ -128,7 +117,6 @@ struct reader {
|
||||
memcpy(file_bbox, r.file_bbox, sizeof(file_bbox));
|
||||
|
||||
geomst = r.geomst;
|
||||
metast = r.metast;
|
||||
|
||||
geom_map = r.geom_map;
|
||||
}
|
||||
|
||||
@@ -324,7 +324,7 @@ struct ordercmp {
|
||||
}
|
||||
} ordercmp;
|
||||
|
||||
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic<long long> *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, unsigned long long label_point, long long extent) {
|
||||
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic<long long> *geompos, FILE **geomfile, const char *fname, signed char t, int layer, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, unsigned long long label_point, long long extent) {
|
||||
if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) {
|
||||
int xo, yo;
|
||||
int span = 1 << (nextzoom - z);
|
||||
@@ -411,19 +411,16 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
|
||||
sf.tippecanoe_minzoom = tippecanoe_minzoom;
|
||||
sf.has_tippecanoe_maxzoom = tippecanoe_maxzoom != -1;
|
||||
sf.tippecanoe_maxzoom = tippecanoe_maxzoom;
|
||||
sf.metapos = metastart;
|
||||
sf.geometry = geom2;
|
||||
sf.index = index;
|
||||
sf.label_point = label_point;
|
||||
sf.extent = extent;
|
||||
sf.feature_minzoom = feature_minzoom;
|
||||
|
||||
if (metastart < 0) {
|
||||
for (size_t i = 0; i < metakeys.size(); i++) {
|
||||
sf.keys.push_back(metakeys[i]);
|
||||
sf.values.push_back(metavals[i]);
|
||||
}
|
||||
}
|
||||
|
||||
serialize_feature(geomfile[j], &sf, &geompos[j], fname, SHIFT_RIGHT(initial_x[segment]), SHIFT_RIGHT(initial_y[segment]), true);
|
||||
}
|
||||
@@ -1283,7 +1280,6 @@ long long choose_minextent(std::vector<long long> &extents, double f) {
|
||||
|
||||
struct write_tile_args {
|
||||
struct task *tasks = NULL;
|
||||
char *metabase = NULL;
|
||||
char *stringpool = NULL;
|
||||
int min_detail = 0;
|
||||
sqlite3 *outdb = NULL;
|
||||
@@ -1306,7 +1302,6 @@ struct write_tile_args {
|
||||
int low_detail = 0;
|
||||
double simplification = 0;
|
||||
std::atomic<long long> *most = NULL;
|
||||
long long *meta_off = NULL;
|
||||
long long *pool_off = NULL;
|
||||
unsigned *initial_x = NULL;
|
||||
unsigned *initial_y = NULL;
|
||||
@@ -1429,9 +1424,9 @@ void remove_attributes(serial_feature &sf, std::set<std::string> const &exclude_
|
||||
}
|
||||
}
|
||||
|
||||
serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, FILE **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time) {
|
||||
serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, FILE **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time) {
|
||||
while (1) {
|
||||
serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y);
|
||||
serial_feature sf = deserialize_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y);
|
||||
if (sf.t < 0) {
|
||||
return sf;
|
||||
}
|
||||
@@ -1460,7 +1455,7 @@ serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, cha
|
||||
|
||||
if (first_time && pass == 0) { /* only write out the next zoom once, even if we retry */
|
||||
if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) {
|
||||
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.label_point, sf.extent);
|
||||
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.label_point, sf.extent);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1563,8 +1558,6 @@ serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, cha
|
||||
struct run_prefilter_args {
|
||||
FILE *geoms = NULL;
|
||||
std::atomic<long long> *geompos_in = NULL;
|
||||
char *metabase = NULL;
|
||||
long long *meta_off = NULL;
|
||||
int z = 0;
|
||||
unsigned tx = 0;
|
||||
unsigned ty = 0;
|
||||
@@ -1600,7 +1593,7 @@ void *run_prefilter(void *v) {
|
||||
json_writer state(rpa->prefilter_fp);
|
||||
|
||||
while (1) {
|
||||
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time);
|
||||
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time);
|
||||
if (sf.t < 0) {
|
||||
break;
|
||||
}
|
||||
@@ -1850,7 +1843,7 @@ void add_sample_to(std::vector<T> &vals, T val, size_t &increment, size_t seq) {
|
||||
}
|
||||
}
|
||||
|
||||
long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) {
|
||||
long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) {
|
||||
double merge_fraction = 1;
|
||||
double mingap_fraction = 1;
|
||||
double minextent_fraction = 1;
|
||||
@@ -1954,8 +1947,6 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
|
||||
rpa.geoms = geoms;
|
||||
rpa.geompos_in = geompos_in;
|
||||
rpa.metabase = metabase;
|
||||
rpa.meta_off = meta_off;
|
||||
rpa.z = z;
|
||||
rpa.tx = tx;
|
||||
rpa.ty = ty;
|
||||
@@ -2003,7 +1994,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
ssize_t which_partial = -1;
|
||||
|
||||
if (prefilter == NULL) {
|
||||
sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time);
|
||||
sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time);
|
||||
} else {
|
||||
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
|
||||
}
|
||||
@@ -2837,7 +2828,7 @@ void *run_thread(void *vargs) {
|
||||
|
||||
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
|
||||
|
||||
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy);
|
||||
long long len = write_tile(geom, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy);
|
||||
|
||||
if (len < 0) {
|
||||
int *err = &arg->err;
|
||||
@@ -2902,7 +2893,7 @@ void *run_thread(void *vargs) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies) {
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies) {
|
||||
last_progress = 0;
|
||||
|
||||
// The existing layermaps are one table per input thread.
|
||||
@@ -3049,7 +3040,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
atomic_strategy strategy;
|
||||
|
||||
for (size_t thread = 0; thread < threads; thread++) {
|
||||
args[thread].metabase = metabase;
|
||||
args[thread].stringpool = stringpool;
|
||||
args[thread].min_detail = min_detail;
|
||||
args[thread].outdb = outdb; // locked with db_lock
|
||||
@@ -3086,7 +3076,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
args[thread].full_detail = full_detail;
|
||||
args[thread].low_detail = low_detail;
|
||||
args[thread].most = &most; // locked with var_lock
|
||||
args[thread].meta_off = meta_off;
|
||||
args[thread].pool_off = pool_off;
|
||||
args[thread].initial_x = initial_x;
|
||||
args[thread].initial_y = initial_y;
|
||||
|
||||
@@ -61,9 +61,9 @@ struct strategy {
|
||||
strategy() = default;
|
||||
};
|
||||
|
||||
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
|
||||
long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
|
||||
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies);
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies);
|
||||
|
||||
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user