diff --git a/geometry.cpp b/geometry.cpp index 30f8fd9e..a0330308 100644 --- a/geometry.cpp +++ b/geometry.cpp @@ -239,7 +239,7 @@ drawvec impose_tile_boundaries(const drawvec &geom, long long extent) { return out; } -drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos) { +drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom) { int res = 1 << (32 - detail - z); long long area = 1LL << (32 - z); @@ -276,12 +276,16 @@ drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool ma d.y += ty * (1LL << (32 - z)); } - // to quadkey struct node n; - n.index = encode_quadkey((unsigned) d.x, (unsigned) d.y); + n.index = encode_vertex((unsigned) d.x, (unsigned) d.y); + size_t bloom_ix = n.index % (shared_nodes_bloom.size() * 8); + unsigned char bloom_mask = 1 << (bloom_ix & 7); + bloom_ix >>= 3; - if (bsearch(&n, shared_nodes_map, nodepos / sizeof(node), sizeof(node), nodecmp) != NULL) { - geom[i].necessary = true; + if (shared_nodes_bloom[bloom_ix] & bloom_mask) { + if (bsearch(&n, shared_nodes_map, nodepos / sizeof(node), sizeof(node), nodecmp) != NULL) { + geom[i].necessary = true; + } } } } diff --git a/geometry.hpp b/geometry.hpp index 997842c7..d85ddcac 100644 --- a/geometry.hpp +++ b/geometry.hpp @@ -81,7 +81,7 @@ drawvec stairstep(drawvec &geom, int z, int detail); bool point_within_tile(long long x, long long y, int z); int quick_check(const long long *bbox, int z, long long buffer); void douglas_peucker(drawvec &geom, int start, int n, double e, size_t kept, size_t retain, bool prevent_simplify_shared_nodes); -drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos); +drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom); drawvec reorder_lines(const drawvec &geom); drawvec fix_polygon(const drawvec &geom); std::vector chop_polygon(std::vector &geoms); diff --git a/main.cpp b/main.cpp index c90e892a..921f811b 100644 --- a/main.cpp +++ b/main.cpp @@ -2070,7 +2070,7 @@ std::pair read_input(std::vector &sources, char *fname, i #endif struct node n; - n.index = encode_quadkey((unsigned) x, (unsigned) y); + n.index = encode_vertex((unsigned) x, (unsigned) y); fwrite_check((char *) &n, sizeof(struct node), 1, readers[0].nodefile, &readers[0].nodepos, "vertices"); } @@ -2084,6 +2084,9 @@ std::pair read_input(std::vector &sources, char *fname, i fprintf(stderr, "Merging nodes \r"); } + std::string shared_nodes_bloom; + shared_nodes_bloom.resize(34567891); // circa 34MB, size nowhere near a power of 2 + // Sort nodes that can't be simplified away; scan the list to remove duplicates FILE *shared_nodes; @@ -2149,6 +2152,15 @@ std::pair read_input(std::vector &sources, char *fname, i fwrite_check((void *) &here, sizeof(here), 1, shared_nodes, &nodepos, "shared nodes"); written = here; + size_t bloom_ix = here.index % (shared_nodes_bloom.size() * 8); + unsigned char bloom_mask = 1 << (bloom_ix & 7); + bloom_ix >>= 3; + + if (shared_nodes_bloom[bloom_ix] & bloom_mask) { + printf("collision at %lld : %zu/%d\n", nodepos / sizeof(here), bloom_ix, bloom_mask); + } + shared_nodes_bloom[bloom_ix] |= bloom_mask; + #if 0 unsigned wx, wy; decode_quadkey(here.index, &wx, &wy); @@ -2770,7 +2782,7 @@ std::pair read_input(std::vector &sources, char *fname, i std::atomic midx(0); std::atomic midy(0); std::vector strategies; - int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz, shared_nodes_map, nodepos, basezoom, droprate, unidecode_data); + int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz, shared_nodes_map, nodepos, shared_nodes_bloom, basezoom, droprate, unidecode_data); if (maxzoom != written) { if (written > minzoom) { diff --git a/projection.cpp b/projection.cpp index f5f71382..9f242d5a 100644 --- a/projection.cpp +++ b/projection.cpp @@ -217,3 +217,7 @@ void set_projection_or_exit(const char *optarg) { exit(EXIT_ARGS); } } + +unsigned long long encode_vertex(unsigned int wx, unsigned int wy) { + return (((unsigned long long) wx) << 32) | wy; +} diff --git a/projection.hpp b/projection.hpp index d649ef56..96b096ed 100644 --- a/projection.hpp +++ b/projection.hpp @@ -26,4 +26,6 @@ void decode_quadkey(unsigned long long index, unsigned *wx, unsigned *wy); unsigned long long encode_hilbert(unsigned int wx, unsigned int wy); void decode_hilbert(unsigned long long index, unsigned *wx, unsigned *wy); +unsigned long long encode_vertex(unsigned int wx, unsigned int wy); + #endif diff --git a/serial.cpp b/serial.cpp index bd694ebb..405d5508 100644 --- a/serial.cpp +++ b/serial.cpp @@ -407,7 +407,7 @@ static void add_scaled_node(struct reader *r, serialization_state *sst, draw g) long long y = SHIFT_LEFT(g.y); struct node n; - n.index = encode_quadkey((unsigned) x, (unsigned) y); + n.index = encode_vertex((unsigned) x, (unsigned) y); fwrite_check((char *) &n, sizeof(struct node), 1, r->nodefile, &r->nodepos, sst->fname); } diff --git a/shared_borders.cpp b/shared_borders.cpp index eec81c89..709c8a21 100644 --- a/shared_borders.cpp +++ b/shared_borders.cpp @@ -357,7 +357,7 @@ bool find_common_edges(std::vector &features, int z, int line_de } if (!(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]) || (z < maxzoom && additional[A_GRID_LOW_ZOOMS]))) { // tx and ty are 0 here because we aren't trying to do anything with the shared_nodes_map - simplified_arcs[ai->second] = simplify_lines(dv, z, 0, 0, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, 4, drawvec(), NULL, 0); + simplified_arcs[ai->second] = simplify_lines(dv, z, 0, 0, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, 4, drawvec(), NULL, 0, ""); } else { simplified_arcs[ai->second] = dv; } diff --git a/tile.cpp b/tile.cpp index 7ac57863..792093f9 100644 --- a/tile.cpp +++ b/tile.cpp @@ -541,6 +541,7 @@ struct simplification_worker_arg { drawvec *shared_nodes; node *shared_nodes_map; size_t nodepos; + std::string const *shared_nodes_bloom; }; // If a polygon has collapsed away to nothing during polygon cleaning, @@ -590,7 +591,7 @@ static drawvec revive_polygon(drawvec &geom, double area, int z, int detail) { // This simplifies the geometry of one feature. It is generally called from the feature_simplification_worker // but is broken out here so that it can be called from earlier in write_tile if coalesced geometries build up // too much in memory. -static double simplify_feature(serial_feature *p, drawvec const &shared_nodes, node *shared_nodes_map, size_t nodepos) { +static double simplify_feature(serial_feature *p, drawvec const &shared_nodes, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom) { drawvec geom = p->geometry; signed char t = p->t; int z = p->z; @@ -637,13 +638,13 @@ static double simplify_feature(serial_feature *p, drawvec const &shared_nodes, n } // continues to simplify to line_detail even if we have extra detail - drawvec ngeom = simplify_lines(geom, z, p->tx, p->ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), p->simplification, t == VT_POLYGON ? 4 : 0, shared_nodes, shared_nodes_map, nodepos); + drawvec ngeom = simplify_lines(geom, z, p->tx, p->ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), p->simplification, t == VT_POLYGON ? 4 : 0, shared_nodes, shared_nodes_map, nodepos, shared_nodes_bloom); if (p->coalesced && prevent[P_SIMPLIFY_SHARED_NODES]) { // do another simplification to eliminate collinearities // that were left behind at the former corners between // coalesced geometries - ngeom = simplify_lines(ngeom, z, p->tx, p->ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), 0.1, t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0); + ngeom = simplify_lines(ngeom, z, p->tx, p->ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), 0.1, t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, ""); } if (t != VT_POLYGON || ngeom.size() >= 3) { @@ -671,7 +672,7 @@ static void *simplification_worker(void *v) { for (size_t i = a->task; i < (*features).size(); i += a->tasks) { double area = 0; if (!a->trying_to_stop_early) { - area = simplify_feature(&((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos); + area = simplify_feature(&((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos, *(a->shared_nodes_bloom)); } signed char t = (*features)[i].t; @@ -896,6 +897,7 @@ struct write_tile_args { bool compressed; node *shared_nodes_map; size_t nodepos; + std::string const *shared_nodes_bloom; std::set const *skip_children; // what is being skipped at this zoom std::set skip_children_out; // what will be skipped in the next zoom }; @@ -1534,7 +1536,7 @@ void skip_tile(decompressor *geoms, std::atomic *geompos_in, bool com } } -long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { +long long write_tile(decompressor *geoms, std::atomic *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic *geompos, int minzoom, int maxzoom, double todo, std::atomic *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic *running, double simplification, std::vector> *layermaps, std::vector> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector const &unidecode_data, long long estimated_complexity, std::set &skip_children_out) { double merge_fraction = 1; double mingap_fraction = 1; double minextent_fraction = 1; @@ -2048,7 +2050,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch // may not be very effective for reducing memory usage. for (; simplified_geometry_through < features.size(); simplified_geometry_through++) { - simplify_feature(&features[simplified_geometry_through], shared_nodes, shared_nodes_map, nodepos); + simplify_feature(&features[simplified_geometry_through], shared_nodes, shared_nodes_map, nodepos, shared_nodes_bloom); if (features[simplified_geometry_through].t == VT_POLYGON) { drawvec to_clean = features[simplified_geometry_through].geometry; @@ -2247,6 +2249,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch args[i].shared_nodes = &shared_nodes; args[i].shared_nodes_map = shared_nodes_map; args[i].nodepos = nodepos; + args[i].shared_nodes_bloom = &shared_nodes_bloom; args[i].trying_to_stop_early = trying_to_stop_early; if (tasks > 1) { @@ -2326,7 +2329,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (!(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]))) { // XXX revisit: why does this not take zoom into account? layer_features[x].geometry = simplify_lines(layer_features[x].geometry, 32, 0, 0, 0, - !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, layer_features[x].t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0); + !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, layer_features[x].t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, ""); } } @@ -2777,7 +2780,7 @@ exit(EXIT_IMPOSSIBLE); skip_tile(&dc, &geompos, arg->compressed); len = 1; } else { - len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, (*arg->unidecode_data), estimated_complexity, arg->skip_children_out); + len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out); } if (pthread_mutex_lock(&var_lock) != 0) { @@ -2843,7 +2846,7 @@ exit(EXIT_IMPOSSIBLE); return err_or_null; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object *filter, std::vector &strategies, int iz, node *shared_nodes_map, size_t nodepos, int basezoom, double droprate, std::vector const &unidecode_data) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object *filter, std::vector &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data) { last_progress = 0; // The existing layermaps are one table per input thread. @@ -3060,6 +3063,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: args[thread].compressed = (z != iz); args[thread].shared_nodes_map = shared_nodes_map; args[thread].nodepos = nodepos; + args[thread].shared_nodes_bloom = &shared_nodes_bloom; args[thread].skip_children = &skip_children; args[thread].skip_children_out.clear(); diff --git a/tile.hpp b/tile.hpp index 4401d10a..f2fab966 100644 --- a/tile.hpp +++ b/tile.hpp @@ -58,7 +58,7 @@ struct strategy { // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, int basezoom, double droprate, std::vector const &unidecode_data); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);