mirror of
https://github.com/felt/tippecanoe.git
synced 2026-10-02 08:25:40 +02:00
Reduce thrashing during feature ingestion and tiling (#56)
* Remove the concept of "separate metadata"
This was an extra level of attribute indirection (features point
to metadata records which point to key and value strings) which was
intended to reduce the size of temporary storage for features with
large numbers of attributes that were also spread across large numbers
of tiles at maxzoom.
For other kinds of features, the extra indirection slowed things down
instead, and, especially when maxzoom guessing was being used, many more
features were having their metadata externalized than could actually
benefit from it.
* Shave a few bytes off temporary files by using more unsigned integers
* Flush stderr after logging progress
* Revert "Shave a few bytes off temporary files by using more unsigned integers"
This reverts commit eef29084ec.
* Limit the size of the string pools and trees to fit in memory
* Add missing #include
* Move the string pool and search tree from mmap to allocated memory
* Sort in allocated rather than mapped memory too
* Also use pread instead of mapping to read in the data to sort
* When the pool gets too big, switch to just the file, not memory
* Switch string pool from memory to disk when memory is 10% full
* Add to-memory versions of the serialization functions
* Crashy work in progress toward compression
* Fix the pointer bug that was causing the crash
* Serialize features into memory rather than straight to disk
* Compress individual features in the temporary files
* Don't need to store the length of the geometry
* Remove per-feature compression; move minzoom back into the object
* Start adding a stream compressor object
* Track file position within fwrite_check()
* Add compressed stream writer functions
* Pull the writing of the serialized feature out to the callers
* Starting toward compression again from a different point
* Hook up more compression functions
* Remove unused code from the other day
* Make enough deflate calls to flush out all the buffered data
* Start on decompression
* Tile number is uncompressed, tile content is compressed
* Work on alternating compressed and uncompressed in decompression
* Closer, but still doesn't work
* Sort of works
* Works until we get to concatenated tiles
* More attempts that don't work
* One bug down
* It made a tileset!
* Handle nonzero initial zooms
* Fix seeking within compressed feature streams
* Tests pass!
* Remove debug spew
* Oops: remember to delete the temporary files so they don't hang around
* Test that fails with the current compression code
* Properly account for bytes read while closing the compressed stream
* Limit the number of warnings about bad label points
* A little more armor when closing decompression
* This time for sure
* A different, less fragile, test that failed previously with compression
* Move feature stream compression to its own file
* Remove now-unused code to deserialize from a file
* Forgot to add the new files
* Remove a little debugging logging
* Add a couple of comments on what it means to be within decompression
* Fix indentation
* Update changelog. Remove stray debugging comment.
This commit is contained in:
Erica Fischer
@@ -25,6 +25,7 @@
|
||||
#include <errno.h>
|
||||
#include <time.h>
|
||||
#include <fcntl.h>
|
||||
#include <zlib.h>
|
||||
#include <sys/wait.h>
|
||||
#include "mvt.hpp"
|
||||
#include "mbtiles.hpp"
|
||||
@@ -40,6 +41,8 @@
|
||||
#include "milo/dtoa_milo.h"
|
||||
#include "evaluator.hpp"
|
||||
#include "errors.hpp"
|
||||
#include "compression.hpp"
|
||||
#include "protozero/varint.hpp"
|
||||
|
||||
extern "C" {
|
||||
#include "jsonpull/jsonpull.h"
|
||||
@@ -51,7 +54,7 @@ extern "C" {
|
||||
|
||||
// Offset coordinates to keep them positive
|
||||
#define COORD_OFFSET (4LL << 32)
|
||||
#define SHIFT_RIGHT(a) ((long long) std::round((double)(a) / (1LL << geometry_scale)))
|
||||
#define SHIFT_RIGHT(a) ((long long) std::round((double) (a) / (1LL << geometry_scale)))
|
||||
|
||||
#define XSTRINGIFY(s) STRINGIFY(s)
|
||||
#define STRINGIFY(s) #s
|
||||
@@ -328,7 +331,7 @@ struct ordercmp {
|
||||
}
|
||||
} ordercmp;
|
||||
|
||||
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic<long long> *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, unsigned long long label_point, long long extent) {
|
||||
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic<long long> *geompos, compressor **geomfile, const char *fname, signed char t, int layer, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, unsigned long long label_point, long long extent) {
|
||||
if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) {
|
||||
int xo, yo;
|
||||
int span = 1 << (nextzoom - z);
|
||||
@@ -398,9 +401,10 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
|
||||
|
||||
{
|
||||
if (!within[j]) {
|
||||
serialize_int(geomfile[j], nextzoom, &geompos[j], fname);
|
||||
serialize_uint(geomfile[j], tx * span + xo, &geompos[j], fname);
|
||||
serialize_uint(geomfile[j], ty * span + yo, &geompos[j], fname);
|
||||
serialize_int(geomfile[j]->fp, nextzoom, &geompos[j], fname);
|
||||
serialize_uint(geomfile[j]->fp, tx * span + xo, &geompos[j], fname);
|
||||
serialize_uint(geomfile[j]->fp, ty * span + yo, &geompos[j], fname);
|
||||
geomfile[j]->begin();
|
||||
within[j] = 1;
|
||||
}
|
||||
|
||||
@@ -415,21 +419,20 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
|
||||
sf.tippecanoe_minzoom = tippecanoe_minzoom;
|
||||
sf.has_tippecanoe_maxzoom = tippecanoe_maxzoom != -1;
|
||||
sf.tippecanoe_maxzoom = tippecanoe_maxzoom;
|
||||
sf.metapos = metastart;
|
||||
sf.geometry = geom2;
|
||||
sf.index = index;
|
||||
sf.label_point = label_point;
|
||||
sf.extent = extent;
|
||||
sf.feature_minzoom = feature_minzoom;
|
||||
|
||||
if (metastart < 0) {
|
||||
for (size_t i = 0; i < metakeys.size(); i++) {
|
||||
sf.keys.push_back(metakeys[i]);
|
||||
sf.values.push_back(metavals[i]);
|
||||
}
|
||||
for (size_t i = 0; i < metakeys.size(); i++) {
|
||||
sf.keys.push_back(metakeys[i]);
|
||||
sf.values.push_back(metavals[i]);
|
||||
}
|
||||
|
||||
serialize_feature(geomfile[j], &sf, &geompos[j], fname, SHIFT_RIGHT(initial_x[segment]), SHIFT_RIGHT(initial_y[segment]), true);
|
||||
std::string feature = serialize_feature(&sf, SHIFT_RIGHT(initial_x[segment]), SHIFT_RIGHT(initial_y[segment]));
|
||||
geomfile[j]->serialize_long_long(feature.size(), &geompos[j], fname);
|
||||
geomfile[j]->fwrite_check(feature.c_str(), sizeof(char), feature.size(), &geompos[j], fname);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1287,14 +1290,13 @@ long long choose_minextent(std::vector<long long> &extents, double f) {
|
||||
|
||||
struct write_tile_args {
|
||||
struct task *tasks = NULL;
|
||||
char *metabase = NULL;
|
||||
char *stringpool = NULL;
|
||||
int min_detail = 0;
|
||||
sqlite3 *outdb = NULL;
|
||||
const char *outdir = NULL;
|
||||
int buffer = 0;
|
||||
const char *fname = NULL;
|
||||
FILE **geomfile = NULL;
|
||||
compressor **geomfile = NULL;
|
||||
double todo = 0;
|
||||
std::atomic<long long> *along = NULL;
|
||||
double gamma = 0;
|
||||
@@ -1310,7 +1312,6 @@ struct write_tile_args {
|
||||
int low_detail = 0;
|
||||
double simplification = 0;
|
||||
std::atomic<long long> *most = NULL;
|
||||
long long *meta_off = NULL;
|
||||
long long *pool_off = NULL;
|
||||
unsigned *initial_x = NULL;
|
||||
unsigned *initial_y = NULL;
|
||||
@@ -1336,6 +1337,8 @@ struct write_tile_args {
|
||||
struct json_object *filter = NULL;
|
||||
std::atomic<size_t> *dropped_count = NULL;
|
||||
atomic_strategy *strategy = NULL;
|
||||
int zoom = -1;
|
||||
bool compressed;
|
||||
};
|
||||
|
||||
bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
|
||||
@@ -1433,18 +1436,40 @@ void remove_attributes(serial_feature &sf, std::set<std::string> const &exclude_
|
||||
}
|
||||
}
|
||||
|
||||
serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, FILE **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time) {
|
||||
serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, compressor **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed) {
|
||||
while (1) {
|
||||
serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y);
|
||||
if (sf.t < 0) {
|
||||
serial_feature sf;
|
||||
std::string s;
|
||||
long long len;
|
||||
|
||||
if (geoms->deserialize_long_long(&len, geompos_in) == 0) {
|
||||
fprintf(stderr, "Unexpected physical EOF in feature stream\n");
|
||||
exit(EXIT_READ);
|
||||
}
|
||||
if (len == 0) {
|
||||
if (compressed) {
|
||||
geoms->end(geompos_in);
|
||||
}
|
||||
|
||||
sf.t = -2;
|
||||
return sf;
|
||||
}
|
||||
|
||||
s.resize(std::abs(len));
|
||||
size_t n = geoms->fread((void *) s.c_str(), sizeof(char), s.size(), geompos_in);
|
||||
if (n != s.size()) {
|
||||
fprintf(stderr, "Short read (%zu for %zu) from geometry\n", n, s.size());
|
||||
exit(EXIT_READ);
|
||||
}
|
||||
|
||||
sf = deserialize_feature(s, z, tx, ty, initial_x, initial_y);
|
||||
|
||||
size_t passes = pass + 1;
|
||||
double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + pass) / passes + z) / (maxzoom + 1) * 1000) / 10;
|
||||
if (progress >= *oprogress + 0.1) {
|
||||
if (!quiet && !quiet_progress && progress_time()) {
|
||||
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
|
||||
fflush(stderr);
|
||||
}
|
||||
if (logger.json_enabled && progress_time()) {
|
||||
logger.progress_tile(progress);
|
||||
@@ -1464,7 +1489,7 @@ serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, cha
|
||||
|
||||
if (first_time && pass == 0) { /* only write out the next zoom once, even if we retry */
|
||||
if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) {
|
||||
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.label_point, sf.extent);
|
||||
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.label_point, sf.extent);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1565,10 +1590,8 @@ serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, cha
|
||||
}
|
||||
|
||||
struct run_prefilter_args {
|
||||
FILE *geoms = NULL;
|
||||
decompressor *geoms = NULL;
|
||||
std::atomic<long long> *geompos_in = NULL;
|
||||
char *metabase = NULL;
|
||||
long long *meta_off = NULL;
|
||||
int z = 0;
|
||||
unsigned tx = 0;
|
||||
unsigned ty = 0;
|
||||
@@ -1585,7 +1608,7 @@ struct run_prefilter_args {
|
||||
long long alongminus = 0;
|
||||
int buffer = 0;
|
||||
int *within = NULL;
|
||||
FILE **geomfile = NULL;
|
||||
compressor **geomfile = NULL;
|
||||
std::atomic<long long> *geompos = NULL;
|
||||
std::atomic<double> *oprogress = NULL;
|
||||
double todo = 0;
|
||||
@@ -1597,6 +1620,7 @@ struct run_prefilter_args {
|
||||
FILE *prefilter_fp = NULL;
|
||||
struct json_object *filter = NULL;
|
||||
bool first_time = false;
|
||||
bool compressed = false;
|
||||
};
|
||||
|
||||
void *run_prefilter(void *v) {
|
||||
@@ -1604,7 +1628,7 @@ void *run_prefilter(void *v) {
|
||||
json_writer state(rpa->prefilter_fp);
|
||||
|
||||
while (1) {
|
||||
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time);
|
||||
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed);
|
||||
if (sf.t < 0) {
|
||||
break;
|
||||
}
|
||||
@@ -1854,7 +1878,7 @@ void add_sample_to(std::vector<T> &vals, T val, size_t &increment, size_t seq) {
|
||||
}
|
||||
}
|
||||
|
||||
long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) {
|
||||
long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy, bool compressed_input) {
|
||||
double merge_fraction = 1;
|
||||
double mingap_fraction = 1;
|
||||
double minextent_fraction = 1;
|
||||
@@ -1926,11 +1950,22 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
}
|
||||
|
||||
if (*geompos_in != og) {
|
||||
if (fseek(geoms, og, SEEK_SET) != 0) {
|
||||
if (compressed_input) {
|
||||
if (geoms->within) {
|
||||
geoms->end(geompos_in);
|
||||
}
|
||||
|
||||
geoms->begin();
|
||||
}
|
||||
|
||||
if (fseek(geoms->fp, og, SEEK_SET) != 0) {
|
||||
perror("fseek geom");
|
||||
exit(EXIT_SEEK);
|
||||
}
|
||||
|
||||
*geompos_in = og;
|
||||
geoms->zs.avail_in = 0;
|
||||
geoms->zs.avail_out = 0;
|
||||
}
|
||||
|
||||
int prefilter_write = -1, prefilter_read = -1;
|
||||
@@ -1958,8 +1993,6 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
|
||||
rpa.geoms = geoms;
|
||||
rpa.geompos_in = geompos_in;
|
||||
rpa.metabase = metabase;
|
||||
rpa.meta_off = meta_off;
|
||||
rpa.z = z;
|
||||
rpa.tx = tx;
|
||||
rpa.ty = ty;
|
||||
@@ -1988,6 +2021,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
rpa.pool_off = pool_off;
|
||||
rpa.filter = filter;
|
||||
rpa.first_time = first_time;
|
||||
rpa.compressed = compressed_input;
|
||||
|
||||
if (pthread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) {
|
||||
perror("pthread_create (prefilter writer)");
|
||||
@@ -2007,7 +2041,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
ssize_t which_partial = -1;
|
||||
|
||||
if (prefilter == NULL) {
|
||||
sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time);
|
||||
sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time, compressed_input);
|
||||
} else {
|
||||
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
|
||||
}
|
||||
@@ -2414,7 +2448,8 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
int j;
|
||||
for (j = 0; j < child_shards; j++) {
|
||||
if (within[j]) {
|
||||
serialize_byte(geomfile[j], -2, &geompos[j], fname);
|
||||
geomfile[j]->serialize_long_long(0, &geompos[j], fname); // EOF
|
||||
geomfile[j]->end(&geompos[j], fname);
|
||||
within[j] = 0;
|
||||
}
|
||||
}
|
||||
@@ -2577,6 +2612,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
if (progress >= oprogress + 0.1) {
|
||||
if (!quiet && !quiet_progress && progress_time()) {
|
||||
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
|
||||
fflush(stderr);
|
||||
}
|
||||
if (logger.json_enabled && progress_time()) {
|
||||
logger.progress_tile(progress);
|
||||
@@ -2672,7 +2708,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
|
||||
std::string pbf = tile.encode();
|
||||
|
||||
if (!prevent[P_TILE_COMPRESSION]) {
|
||||
compress(pbf, compressed);
|
||||
compress(pbf, compressed, true);
|
||||
} else {
|
||||
compressed = pbf;
|
||||
}
|
||||
@@ -2818,7 +2854,15 @@ void *run_thread(void *vargs) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// printf("%lld of geom_size\n", (long long) geom_size[j]);
|
||||
// If this is zoom level 0, the geomfd will be uncompressed data,
|
||||
// because (at least for now) it needs to stay uncompressed during
|
||||
// the sort and post-sort maxzoom calculation and fixup so that
|
||||
// the sort can rearrange individual features and the fixup can
|
||||
// then adjust their minzooms without decompressing and recompressing
|
||||
// each feature.
|
||||
//
|
||||
// In higher zooms, it will be compressed data written out during the
|
||||
// previous zoom.
|
||||
|
||||
FILE *geom = fdopen(arg->geomfd[j], "rb");
|
||||
if (geom == NULL) {
|
||||
@@ -2826,6 +2870,8 @@ void *run_thread(void *vargs) {
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
|
||||
decompressor dc(geom);
|
||||
|
||||
std::atomic<long long> geompos(0);
|
||||
long long prevgeom = 0;
|
||||
|
||||
@@ -2833,17 +2879,31 @@ void *run_thread(void *vargs) {
|
||||
int z;
|
||||
unsigned x, y;
|
||||
|
||||
if (!deserialize_int_io(geom, &z, &geompos)) {
|
||||
// These z/x/y are uncompressed so we can seek to the start of the
|
||||
// compressed feature data that immediately follows.
|
||||
|
||||
if (!dc.deserialize_int(&z, &geompos)) {
|
||||
break;
|
||||
}
|
||||
deserialize_uint_io(geom, &x, &geompos);
|
||||
deserialize_uint_io(geom, &y, &geompos);
|
||||
dc.deserialize_uint(&x, &geompos);
|
||||
dc.deserialize_uint(&y, &geompos);
|
||||
#if 0
|
||||
// currently broken because also requires tracking nextzoom when skipping zooms
|
||||
if (z != arg->zoom) {
|
||||
fprintf(stderr, "Expected zoom %d, found zoom %d\n", arg->zoom, z);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (arg->compressed) {
|
||||
dc.begin();
|
||||
}
|
||||
|
||||
arg->wrote_zoom = z;
|
||||
|
||||
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
|
||||
|
||||
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy);
|
||||
long long len = write_tile(&dc, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed);
|
||||
|
||||
if (len < 0) {
|
||||
int *err = &arg->err;
|
||||
@@ -2908,7 +2968,7 @@ void *run_thread(void *vargs) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies) {
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz) {
|
||||
last_progress = 0;
|
||||
|
||||
// The existing layermaps are one table per input thread.
|
||||
@@ -2933,10 +2993,11 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
}
|
||||
|
||||
int z;
|
||||
for (z = 0; z <= maxzoom; z++) {
|
||||
for (z = iz; z <= maxzoom; z++) {
|
||||
std::atomic<long long> most(0);
|
||||
|
||||
FILE *sub[TEMP_FILES];
|
||||
compressor compressors[TEMP_FILES];
|
||||
compressor *sub[TEMP_FILES];
|
||||
int subfd[TEMP_FILES];
|
||||
for (size_t j = 0; j < TEMP_FILES; j++) {
|
||||
char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX" XSTRINGIFY(INT_MAX)) + 1];
|
||||
@@ -2947,11 +3008,13 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
perror(geomname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
sub[j] = fopen_oflag(geomname, "wb", O_WRONLY | O_CLOEXEC);
|
||||
if (sub[j] == NULL) {
|
||||
FILE *fp = fopen_oflag(geomname, "wb", O_WRONLY | O_CLOEXEC);
|
||||
if (fp == NULL) {
|
||||
perror(geomname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
compressors[j] = compressor(fp);
|
||||
sub[j] = &compressors[j];
|
||||
unlink(geomname);
|
||||
}
|
||||
|
||||
@@ -2970,7 +3033,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
}
|
||||
// XXX is it useful to divide further if we know we are skipping
|
||||
// some zoom levels? Is it faster to have fewer CPUs working on
|
||||
// sharding, but more deeply, or fewer CPUs, less deeply?
|
||||
// sharding, but more deeply, or more CPUs, less deeply?
|
||||
if (threads > useful_threads) {
|
||||
threads = useful_threads;
|
||||
}
|
||||
@@ -3055,7 +3118,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
atomic_strategy strategy;
|
||||
|
||||
for (size_t thread = 0; thread < threads; thread++) {
|
||||
args[thread].metabase = metabase;
|
||||
args[thread].stringpool = stringpool;
|
||||
args[thread].min_detail = min_detail;
|
||||
args[thread].outdb = outdb; // locked with db_lock
|
||||
@@ -3092,7 +3154,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
args[thread].full_detail = full_detail;
|
||||
args[thread].low_detail = low_detail;
|
||||
args[thread].most = &most; // locked with var_lock
|
||||
args[thread].meta_off = meta_off;
|
||||
args[thread].pool_off = pool_off;
|
||||
args[thread].initial_x = initial_x;
|
||||
args[thread].initial_y = initial_y;
|
||||
@@ -3110,6 +3171,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
args[thread].wrote_zoom = -1;
|
||||
args[thread].still_dropping = false;
|
||||
args[thread].strategy = &strategy;
|
||||
args[thread].zoom = z;
|
||||
args[thread].compressed = (z != iz);
|
||||
|
||||
if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) {
|
||||
perror("pthread_create");
|
||||
@@ -3188,7 +3251,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
|
||||
exit(EXIT_CLOSE);
|
||||
}
|
||||
}
|
||||
if (fclose(sub[j]) != 0) {
|
||||
if (sub[j]->fclose() != 0) {
|
||||
perror("close subfile");
|
||||
exit(EXIT_CLOSE);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user