diff --git a/main.cpp b/main.cpp index 7d38cd81..5b053c96 100644 --- a/main.cpp +++ b/main.cpp @@ -99,6 +99,7 @@ int retain_points_multiplier = 1; std::vector unidecode_data; size_t maximum_string_attribute_length = 0; std::string accumulate_numeric; +const char *tmpdir = "/tmp"; std::vector order_by; bool order_reverse; @@ -740,7 +741,7 @@ void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::ato parser_created = true; } -void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int splits, long long mem, const char *tmpdir, long long *availfiles, FILE *geomfile, FILE *indexfile, std::atomic *geompos_out, long long *progress, long long *progress_max, long long *progress_reported, int maxzoom, int basezoom, double droprate, double gamma, struct drop_state *ds) { +void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int splits, long long mem, long long *availfiles, FILE *geomfile, FILE *indexfile, std::atomic *geompos_out, long long *progress, long long *progress_max, long long *progress_reported, int maxzoom, int basezoom, double droprate, double gamma, struct drop_state *ds) { // Arranged as bits to facilitate subdividing again if a subdivided file is still huge int splitbits = log(splits) / log(2); splits = 1 << splitbits; @@ -1028,7 +1029,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split // counter backward but will be an honest estimate of the work remaining. *progress_max += geomst.st_size / 4; - radix1(&geomfds[i], &indexfds[i], 1, prefix + splitbits, *availfiles / 4, mem, tmpdir, availfiles, geomfile, indexfile, geompos_out, progress, progress_max, progress_reported, maxzoom, basezoom, droprate, gamma, ds); + radix1(&geomfds[i], &indexfds[i], 1, prefix + splitbits, *availfiles / 4, mem, availfiles, geomfile, indexfile, geompos_out, progress, progress_max, progress_reported, maxzoom, basezoom, droprate, gamma, ds); already_closed = 1; } } @@ -1063,7 +1064,7 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d } } -void radix(std::vector &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic *geompos, int maxzoom, int basezoom, double droprate, double gamma) { +void radix(std::vector &readers, int nreaders, FILE *geomfile, FILE *indexfile, std::atomic *geompos, int maxzoom, int basezoom, double droprate, double gamma) { // Run through the index and geometry for each reader, // splitting the contents out by index into as many // sub-files as we can write to simultaneously. @@ -1113,7 +1114,7 @@ void radix(std::vector &readers, int nreaders, FILE *geomfile, FI long long progress = 0, progress_max = geom_total, progress_reported = -1; long long availfiles_before = availfiles; - radix1(geomfds, indexfds, nreaders, 0, splits, mem, tmpdir, &availfiles, geomfile, indexfile, geompos, &progress, &progress_max, &progress_reported, maxzoom, basezoom, droprate, gamma, ds); + radix1(geomfds, indexfds, nreaders, 0, splits, mem, &availfiles, geomfile, indexfile, geompos, &progress, &progress_max, &progress_reported, maxzoom, basezoom, droprate, gamma, ds); if (availfiles - 2 * nreaders != availfiles_before) { fprintf(stderr, "Internal error: miscounted available file descriptors: %lld vs %lld\n", availfiles - 2 * nreaders, availfiles); @@ -1214,7 +1215,7 @@ double round_droprate(double r) { return std::round(r * 100000.0) / 100000.0; } -std::pair read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::unordered_map const *attribute_types, const char *pgm, std::unordered_map const *attribute_accum, std::map const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) { +std::pair read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, json_object *filter, double droprate, int buffer, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::unordered_map const *attribute_types, const char *pgm, std::unordered_map const *attribute_accum, std::map const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) { int ret = EXIT_SUCCESS; std::vector readers; @@ -2207,7 +2208,7 @@ std::pair read_input(std::vector &sources, char *fname, i serialize_uint(geomfile, ix, &geompos, fname); serialize_uint(geomfile, iy, &geompos, fname); - radix(readers, CPUS, geomfile, indexfile, tmpdir, &geompos, maxzoom, basezoom, droprate, gamma); + radix(readers, CPUS, geomfile, indexfile, &geompos, maxzoom, basezoom, droprate, gamma); /* end of tile */ serialize_ulong_long(geomfile, 0, &geompos, fname); // EOF @@ -2755,7 +2756,7 @@ std::pair read_input(std::vector &sources, char *fname, i std::atomic midx(0); std::atomic midy(0); std::vector strategies; - int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz, shared_nodes_map, nodepos, shared_nodes_bloom, basezoom, droprate, unidecode_data); + int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz, shared_nodes_map, nodepos, shared_nodes_bloom, basezoom, droprate, unidecode_data); if (maxzoom != written) { if (written > minzoom) { @@ -2991,7 +2992,6 @@ int main(int argc, char **argv) { double droprate = 2.5; double gamma = 0; int buffer = 5; - const char *tmpdir = "/tmp"; const char *attribution = NULL; std::vector sources; const char *prefilter = NULL; @@ -3833,7 +3833,7 @@ int main(int argc, char **argv) { auto input_ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_dir, - maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom); + maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom); ret = std::get<0>(input_ret); diff --git a/main.hpp b/main.hpp index 714af5c3..08dad065 100644 --- a/main.hpp +++ b/main.hpp @@ -54,6 +54,7 @@ extern int retain_points_multiplier; extern size_t maximum_string_attribute_length; extern std::string accumulate_numeric; extern unsigned long long preserve_multiplier_density_threshold; +extern const char *tmpdir; struct order_field { std::string name; diff --git a/tile.cpp b/tile.cpp index 22ef2eb2..68f65020 100644 --- a/tile.cpp +++ b/tile.cpp @@ -904,6 +904,7 @@ struct write_tile_args { atomic_strategy *strategy = NULL; int zoom = -1; bool compressed; + bool first_zoom; node *shared_nodes_map; size_t nodepos; std::string const *shared_nodes_bloom; @@ -1063,7 +1064,7 @@ struct next_feature_state { // This function is called repeatedly from write_tile() to retrieve the next feature // from the input stream. If the stream is at an end, it returns a feature with the // geometry type set to -2. -static serial_feature next_feature(decompressor *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, std::atomic *within, compressor **geomfile, std::atomic *geompos, long long start_geompos[], std::atomic *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr &tile_stringpool, std::vector const &unidecode_data, next_feature_state &next_feature_state, double droprate) { +static serial_feature next_feature(decompressor *geoms, std::atomic *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic *along, long long alongminus, int buffer, std::atomic *within, compressor **geomfile, std::atomic *geompos, long long start_geompos[], std::atomic *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector> *layer_unmaps, bool first_time, bool compressed, bool first_zoom, multiplier_state *multiplier_state, std::shared_ptr &tile_stringpool, std::vector const &unidecode_data, next_feature_state &next_feature_state, double droprate) { double extra_multiplier_zooms = log(retain_points_multiplier) / log(droprate); while (1) { @@ -1123,9 +1124,9 @@ static serial_feature next_feature(decompressor *geoms, std::atomic * sf = deserialize_feature(s, z, tx, ty, initial_x, initial_y); sf.stringpool = global_stringpool + pool_off[sf.segment]; - if (!next_feature_state.doing_deferrals && additional[A_DISTINGUISH_DUPLICATES] && sf.index == next_feature_state.previndex) { + if (!next_feature_state.doing_deferrals && first_zoom && additional[A_DISTINGUISH_DUPLICATES] && sf.index == next_feature_state.previndex) { if (next_feature_state.which_deferral >= next_feature_state.deferrals.size()) { - std::string tmpname = "/tmp/XXXXXXXXXX"; + std::string tmpname = std::string(tmpdir) + std::string("/deferralXXXXXXXXXX"); int fd = mkstemp((char *) tmpname.c_str()); if (fd < 0) { fprintf(stderr, "Can't create temporary file %zu for feature deferral: %s\n", next_feature_state.which_deferral, strerror(errno)); @@ -1142,7 +1143,13 @@ static serial_feature next_feature(decompressor *geoms, std::atomic * fwrite(&len, sizeof(len), 1, next_feature_state.deferrals[next_feature_state.which_deferral]); fwrite(s.c_str(), 1, len, next_feature_state.deferrals[next_feature_state.which_deferral]); - next_feature_state.which_deferral++; + + // limit the maximum depth of deferral, + // because even at z0 we will run out of file descriptors + // (and tile space) if there are large numbers of duplicates + if (next_feature_state.which_deferral < 50) { + next_feature_state.which_deferral++; + } continue; } else { next_feature_state.which_deferral = 0; @@ -1369,6 +1376,7 @@ struct run_prefilter_args { std::vector const *unidecode_data; bool first_time = false; bool compressed = false; + bool first_zoom = false; double droprate = 1; }; @@ -1380,7 +1388,7 @@ void *run_prefilter(void *v) { next_feature_state next_feature_state; while (1) { - serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->start_geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->global_stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed, &multiplier_state, tile_stringpool, *(rpa->unidecode_data), next_feature_state, rpa->droprate); + serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->start_geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->global_stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed, rpa->first_zoom, &multiplier_state, tile_stringpool, *(rpa->unidecode_data), next_feature_state, rpa->droprate); if (sf.t < 0) { break; } @@ -1676,7 +1684,7 @@ void skip_tile(decompressor *geoms, std::atomic *geompos_in, bool com } } -long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { +long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, bool first_zoom, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { double merge_fraction = 1; double mingap_fraction = 1; double minextent_fraction = 1; @@ -1855,6 +1863,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch rpa.unidecode_data = &unidecode_data; rpa.first_time = first_time; rpa.compressed = compressed_input; + rpa.first_zoom = first_zoom; rpa.droprate = arg->droprate; // this does need to be a real thread, so we can pipe both to and from it @@ -1887,7 +1896,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch ssize_t which_serial_feature = -1; if (prefilter == NULL) { - sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, start_geompos, &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, compressed_input, &multiplier_state, tile_stringpool, unidecode_data, next_feature_state, arg->droprate); + sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, start_geompos, &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, compressed_input, first_zoom, &multiplier_state, tile_stringpool, unidecode_data, next_feature_state, arg->droprate); } else { sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL, key_pool); } @@ -3075,7 +3084,7 @@ exit(EXIT_IMPOSSIBLE); len = 1; } else { arg->wrote_zoom = z; - len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out); + len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->first_zoom, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out); } if (pthread_mutex_lock(&var_lock) != 0) { @@ -3141,7 +3150,7 @@ exit(EXIT_IMPOSSIBLE); return err_or_null; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object *filter, std::vector &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object *filter, std::vector &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data) { last_progress = 0; // The existing layermaps are one table per input thread. @@ -3333,6 +3342,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: args[thread].strategy = &strategy; args[thread].zoom = z; args[thread].compressed = (z != iz); + args[thread].first_zoom = (z == iz); args[thread].shared_nodes_map = shared_nodes_map; args[thread].nodepos = nodepos; args[thread].shared_nodes_bloom = &shared_nodes_bloom; diff --git a/tile.hpp b/tile.hpp index 5735d041..dbf5eb51 100644 --- a/tile.hpp +++ b/tile.hpp @@ -62,7 +62,7 @@ struct strategy { // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);