diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Put a bloom filter in front of the binary search for shared nodes

Browse Source
This commit is contained in:
Erica Fischer
2024-08-05 13:37:40 -07:00
parent 53ef0a9984
commit 60ad06a9af
9 changed files with 46 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
geometry.cpp
+9 -5
View File
@@ -239,7 +239,7 @@ drawvec impose_tile_boundaries(const drawvec &geom, long long extent) {
return out;
}
drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos) {
drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom) {
int res = 1 << (32 - detail - z);
long long area = 1LL << (32 - z);
@@ -276,12 +276,16 @@ drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool ma
d.y += ty * (1LL << (32 - z));
}
// to quadkey
struct node n;
n.index = encode_quadkey((unsigned) d.x, (unsigned) d.y);
n.index = encode_vertex((unsigned) d.x, (unsigned) d.y);
size_t bloom_ix = n.index % (shared_nodes_bloom.size() * 8);
unsigned char bloom_mask = 1 << (bloom_ix & 7);
bloom_ix >>= 3;
if (bsearch(&n, shared_nodes_map, nodepos / sizeof(node), sizeof(node), nodecmp) != NULL) {
geom[i].necessary = true;
if (shared_nodes_bloom[bloom_ix] & bloom_mask) {
if (bsearch(&n, shared_nodes_map, nodepos / sizeof(node), sizeof(node), nodecmp) != NULL) {
geom[i].necessary = true;
}
}
}
}
geometry.hpp
+1 -1
View File
@@ -81,7 +81,7 @@ drawvec stairstep(drawvec &geom, int z, int detail);
bool point_within_tile(long long x, long long y, int z);
int quick_check(const long long *bbox, int z, long long buffer);
void douglas_peucker(drawvec &geom, int start, int n, double e, size_t kept, size_t retain, bool prevent_simplify_shared_nodes);
drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos);
drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom);
drawvec reorder_lines(const drawvec &geom);
drawvec fix_polygon(const drawvec &geom);
std::vector<drawvec> chop_polygon(std::vector<drawvec> &geoms);
main.cpp
+14 -2
View File
@@ -2070,7 +2070,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
#endif
struct node n;
n.index = encode_quadkey((unsigned) x, (unsigned) y);
n.index = encode_vertex((unsigned) x, (unsigned) y);
fwrite_check((char *) &n, sizeof(struct node), 1, readers[0].nodefile, &readers[0].nodepos, "vertices");
}
@@ -2084,6 +2084,9 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
fprintf(stderr, "Merging nodes \r");
}
std::string shared_nodes_bloom;
shared_nodes_bloom.resize(34567891); // circa 34MB, size nowhere near a power of 2
// Sort nodes that can't be simplified away; scan the list to remove duplicates
FILE *shared_nodes;
@@ -2149,6 +2152,15 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
fwrite_check((void *) &here, sizeof(here), 1, shared_nodes, &nodepos, "shared nodes");
written = here;
size_t bloom_ix = here.index % (shared_nodes_bloom.size() * 8);
unsigned char bloom_mask = 1 << (bloom_ix & 7);
bloom_ix >>= 3;
if (shared_nodes_bloom[bloom_ix] & bloom_mask) {
printf("collision at %lld : %zu/%d\n", nodepos / sizeof(here), bloom_ix, bloom_mask);
}
shared_nodes_bloom[bloom_ix] |= bloom_mask;
#if 0
unsigned wx, wy;
decode_quadkey(here.index, &wx, &wy);
@@ -2770,7 +2782,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
std::atomic<unsigned> midx(0);
std::atomic<unsigned> midy(0);
std::vector<strategy> strategies;
int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz, shared_nodes_map, nodepos, basezoom, droprate, unidecode_data);
int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz, shared_nodes_map, nodepos, shared_nodes_bloom, basezoom, droprate, unidecode_data);
if (maxzoom != written) {
if (written > minzoom) {
projection.cpp
+4
View File
@@ -217,3 +217,7 @@ void set_projection_or_exit(const char *optarg) {
exit(EXIT_ARGS);
}
}
unsigned long long encode_vertex(unsigned int wx, unsigned int wy) {
return (((unsigned long long) wx) << 32) | wy;
}
projection.hpp
+2
View File
@@ -26,4 +26,6 @@ void decode_quadkey(unsigned long long index, unsigned *wx, unsigned *wy);
unsigned long long encode_hilbert(unsigned int wx, unsigned int wy);
void decode_hilbert(unsigned long long index, unsigned *wx, unsigned *wy);
unsigned long long encode_vertex(unsigned int wx, unsigned int wy);
#endif
serial.cpp
+1 -1
View File
@@ -407,7 +407,7 @@ static void add_scaled_node(struct reader *r, serialization_state *sst, draw g)
long long y = SHIFT_LEFT(g.y);
struct node n;
n.index = encode_quadkey((unsigned) x, (unsigned) y);
n.index = encode_vertex((unsigned) x, (unsigned) y);
fwrite_check((char *) &n, sizeof(struct node), 1, r->nodefile, &r->nodepos, sst->fname);
}
shared_borders.cpp
+1 -1
View File
@@ -357,7 +357,7 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
}
if (!(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]) || (z < maxzoom && additional[A_GRID_LOW_ZOOMS]))) {
// tx and ty are 0 here because we aren't trying to do anything with the shared_nodes_map
simplified_arcs[ai->second] = simplify_lines(dv, z, 0, 0, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, 4, drawvec(), NULL, 0);
simplified_arcs[ai->second] = simplify_lines(dv, z, 0, 0, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, 4, drawvec(), NULL, 0, "");
} else {
simplified_arcs[ai->second] = dv;
}
tile.cpp
+13 -9
View File
@@ -541,6 +541,7 @@ struct simplification_worker_arg {
drawvec *shared_nodes;
node *shared_nodes_map;
size_t nodepos;
std::string const *shared_nodes_bloom;
};
// If a polygon has collapsed away to nothing during polygon cleaning,
@@ -590,7 +591,7 @@ static drawvec revive_polygon(drawvec &geom, double area, int z, int detail) {
// This simplifies the geometry of one feature. It is generally called from the feature_simplification_worker
// but is broken out here so that it can be called from earlier in write_tile if coalesced geometries build up
// too much in memory.
static double simplify_feature(serial_feature *p, drawvec const &shared_nodes, node *shared_nodes_map, size_t nodepos) {
static double simplify_feature(serial_feature *p, drawvec const &shared_nodes, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom) {
drawvec geom = p->geometry;
signed char t = p->t;
int z = p->z;
@@ -637,13 +638,13 @@ static double simplify_feature(serial_feature *p, drawvec const &shared_nodes, n
}
// continues to simplify to line_detail even if we have extra detail
drawvec ngeom = simplify_lines(geom, z, p->tx, p->ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), p->simplification, t == VT_POLYGON ? 4 : 0, shared_nodes, shared_nodes_map, nodepos);
drawvec ngeom = simplify_lines(geom, z, p->tx, p->ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), p->simplification, t == VT_POLYGON ? 4 : 0, shared_nodes, shared_nodes_map, nodepos, shared_nodes_bloom);
if (p->coalesced && prevent[P_SIMPLIFY_SHARED_NODES]) {
// do another simplification to eliminate collinearities
// that were left behind at the former corners between
// coalesced geometries
ngeom = simplify_lines(ngeom, z, p->tx, p->ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), 0.1, t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0);
ngeom = simplify_lines(ngeom, z, p->tx, p->ty, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), 0.1, t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, "");
}
if (t != VT_POLYGON || ngeom.size() >= 3) {
@@ -671,7 +672,7 @@ static void *simplification_worker(void *v) {
for (size_t i = a->task; i < (*features).size(); i += a->tasks) {
double area = 0;
if (!a->trying_to_stop_early) {
area = simplify_feature(&((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos);
area = simplify_feature(&((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos, *(a->shared_nodes_bloom));
}
signed char t = (*features)[i].t;
@@ -896,6 +897,7 @@ struct write_tile_args {
bool compressed;
node *shared_nodes_map;
size_t nodepos;
std::string const *shared_nodes_bloom;
std::set<zxy> const *skip_children; // what is being skipped at this zoom
std::set<zxy> skip_children_out; // what will be skipped in the next zoom
};
@@ -1534,7 +1536,7 @@ void skip_tile(decompressor *geoms, std::atomic<long long> *geompos_in, bool com
}
}
long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::vector<std::string> const &unidecode_data, long long estimated_complexity, std::set<zxy> &skip_children_out) {
long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector<std::string> const &unidecode_data, long long estimated_complexity, std::set<zxy> &skip_children_out) {
double merge_fraction = 1;
double mingap_fraction = 1;
double minextent_fraction = 1;
@@ -2048,7 +2050,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// may not be very effective for reducing memory usage.
for (; simplified_geometry_through < features.size(); simplified_geometry_through++) {
simplify_feature(&features[simplified_geometry_through], shared_nodes, shared_nodes_map, nodepos);
simplify_feature(&features[simplified_geometry_through], shared_nodes, shared_nodes_map, nodepos, shared_nodes_bloom);
if (features[simplified_geometry_through].t == VT_POLYGON) {
drawvec to_clean = features[simplified_geometry_through].geometry;
@@ -2247,6 +2249,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
args[i].shared_nodes = &shared_nodes;
args[i].shared_nodes_map = shared_nodes_map;
args[i].nodepos = nodepos;
args[i].shared_nodes_bloom = &shared_nodes_bloom;
args[i].trying_to_stop_early = trying_to_stop_early;
if (tasks > 1) {
@@ -2326,7 +2329,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (!(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]))) {
// XXX revisit: why does this not take zoom into account?
layer_features[x].geometry = simplify_lines(layer_features[x].geometry, 32, 0, 0, 0,
!(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, layer_features[x].t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0);
!(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, layer_features[x].t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, "");
}
}
@@ -2777,7 +2780,7 @@ exit(EXIT_IMPOSSIBLE);
skip_tile(&dc, &geompos, arg->compressed);
len = 1;
} else {
len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, (*arg->unidecode_data), estimated_complexity, arg->skip_children_out);
len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out);
}
if (pthread_mutex_lock(&var_lock) != 0) {
@@ -2843,7 +2846,7 @@ exit(EXIT_IMPOSSIBLE);
return err_or_null;
}
int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, json_object *filter, std::vector<strategy> &strategies, int iz, node *shared_nodes_map, size_t nodepos, int basezoom, double droprate, std::vector<std::string> const &unidecode_data) {
int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, json_object *filter, std::vector<strategy> &strategies, int iz, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector<std::string> const &unidecode_data) {
last_progress = 0;
// The existing layermaps are one table per input thread.
@@ -3060,6 +3063,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
args[thread].compressed = (z != iz);
args[thread].shared_nodes_map = shared_nodes_map;
args[thread].nodepos = nodepos;
args[thread].shared_nodes_bloom = &shared_nodes_bloom;
args[thread].skip_children = &skip_children;
args[thread].skip_children_out.clear();
tile.hpp
+1 -1
View File
@@ -58,7 +58,7 @@ struct strategy {
// long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, int basezoom, double droprate, std::vector<std::string> const &unidecode_data);
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector<std::string> const &unidecode_data);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);