mirror of
https://github.com/felt/tippecanoe.git
synced 2026-10-02 08:25:40 +02:00
FSL-style expressions can use unidecode data to smash case and diacritics (#197)
* Read unidecode data, do some plumbing of it * More unidecode plumbing * Do the unidecode smashing, but it doesn't seem to be working * Ah, that's better! * Add missing header * And reorder the includes too * Shortcut when there is no unidecode data to work with * Update version and changelog * Avoid repeated unidecode smashing of the same constant string
This commit is contained in:
Erica Fischer
@@ -902,6 +902,7 @@ struct write_tile_args {
|
||||
int wrote_zoom = 0;
|
||||
size_t tiling_seg = 0;
|
||||
json_object *filter = NULL;
|
||||
std::vector<std::string> const *unidecode_data;
|
||||
std::atomic<size_t> *dropped_count = NULL;
|
||||
atomic_strategy *strategy = NULL;
|
||||
int zoom = -1;
|
||||
@@ -1027,7 +1028,7 @@ struct multiplier_state {
|
||||
// This function is called repeatedly from write_tile() to retrieve the next feature
|
||||
// from the input stream. If the stream is at an end, it returns a feature with the
|
||||
// geometry type set to -2.
|
||||
static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, compressor **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr<std::string> &tile_stringpool) {
|
||||
static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, compressor **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr<std::string> &tile_stringpool, std::vector<std::string> const &unidecode_data) {
|
||||
while (1) {
|
||||
serial_feature sf;
|
||||
long long len;
|
||||
@@ -1148,7 +1149,7 @@ static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *
|
||||
|
||||
attributes.insert(std::pair<std::string, mvt_value>("$zoom", v2));
|
||||
|
||||
if (!evaluate(attributes, layername, filter, exclude_attributes)) {
|
||||
if (!evaluate(attributes, layername, filter, exclude_attributes, unidecode_data)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1236,6 +1237,7 @@ struct run_prefilter_args {
|
||||
long long *pool_off = NULL;
|
||||
FILE *prefilter_fp = NULL;
|
||||
json_object *filter = NULL;
|
||||
std::vector<std::string> const *unidecode_data;
|
||||
bool first_time = false;
|
||||
bool compressed = false;
|
||||
};
|
||||
@@ -1247,7 +1249,7 @@ void *run_prefilter(void *v) {
|
||||
std::shared_ptr<std::string> tile_stringpool = std::make_shared<std::string>();
|
||||
|
||||
while (1) {
|
||||
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->global_stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed, &multiplier_state, tile_stringpool);
|
||||
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->global_stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed, &multiplier_state, tile_stringpool, *(rpa->unidecode_data));
|
||||
if (sf.t < 0) {
|
||||
break;
|
||||
}
|
||||
@@ -1465,7 +1467,7 @@ return;
|
||||
}
|
||||
}
|
||||
|
||||
long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy, bool compressed_input, node *shared_nodes_map, size_t nodepos) {
|
||||
long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::vector<std::string> const &unidecode_data) {
|
||||
double merge_fraction = 1;
|
||||
double mingap_fraction = 1;
|
||||
double minextent_fraction = 1;
|
||||
@@ -1623,6 +1625,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
|
||||
rpa.global_stringpool = global_stringpool;
|
||||
rpa.pool_off = pool_off;
|
||||
rpa.filter = filter;
|
||||
rpa.unidecode_data = &unidecode_data;
|
||||
rpa.first_time = first_time;
|
||||
rpa.compressed = compressed_input;
|
||||
|
||||
@@ -1651,7 +1654,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
|
||||
ssize_t which_serial_feature = -1;
|
||||
|
||||
if (prefilter == NULL) {
|
||||
sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, compressed_input, &multiplier_state, tile_stringpool);
|
||||
sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, compressed_input, &multiplier_state, tile_stringpool, unidecode_data);
|
||||
} else {
|
||||
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
|
||||
}
|
||||
@@ -2575,7 +2578,7 @@ exit(EXIT_IMPOSSIBLE);
|
||||
|
||||
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
|
||||
|
||||
long long len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos);
|
||||
long long len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, (*arg->unidecode_data));
|
||||
|
||||
if (pthread_mutex_lock(&var_lock) != 0) {
|
||||
perror("pthread_mutex_lock");
|
||||
@@ -2640,7 +2643,7 @@ exit(EXIT_IMPOSSIBLE);
|
||||
return err_or_null;
|
||||
}
|
||||
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, json_object *filter, std::vector<strategy> &strategies, int iz, node *shared_nodes_map, size_t nodepos, int basezoom, double droprate) {
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, json_object *filter, std::vector<strategy> &strategies, int iz, node *shared_nodes_map, size_t nodepos, int basezoom, double droprate, std::vector<std::string> const &unidecode_data) {
|
||||
last_progress = 0;
|
||||
|
||||
// The existing layermaps are one table per input thread.
|
||||
@@ -2838,6 +2841,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
|
||||
args[thread].postfilter = postfilter;
|
||||
args[thread].attribute_accum = attribute_accum;
|
||||
args[thread].filter = filter;
|
||||
args[thread].unidecode_data = &unidecode_data;
|
||||
|
||||
args[thread].tasks = dispatches[thread].tasks;
|
||||
args[thread].running = &running;
|
||||
|
||||
Reference in New Issue
Block a user