diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

More plumbing for key_pool

Browse Source
This commit is contained in:
Erica Fischer
2024-11-04 13:45:24 -08:00
parent ab95915f24
commit 767017408d
7 changed files with 21 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
geobuf.cpp
+5 -4
View File
@@ -270,14 +270,14 @@ std::vector<drawvec_type> readGeometry(protozero::pbf_reader &pbf, size_t dim, d
return ret; return ret;
} }
void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername) { void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<std::string> &keys, struct serialization_state *sst, int layer, std::string layername, key_pool &key_pool) {
std::vector<drawvec_type> dv; std::vector<drawvec_type> dv;
long long id = 0; long long id = 0;
bool has_id = false; bool has_id = false;
std::vector<serial_val> values; std::vector<serial_val> values;
std::map<std::string, serial_val> other; std::map<std::string, serial_val> other;
std::vector<std::string> full_keys; std::vector<std::shared_ptr<std::string>> full_keys;
std::vector<serial_val> full_values; std::vector<serial_val> full_values;
while (pbf.next()) { while (pbf.next()) {
@@ -338,7 +338,7 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<s
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
full_keys.push_back(keys[properties[i]]); full_keys.push_back(key_pool.pool(keys[properties[i]]));
full_values.push_back(values[properties[i + 1]]); full_values.push_back(values[properties[i + 1]]);
} }
@@ -434,10 +434,11 @@ struct queue_run_arg {
void *run_parse_feature(void *v) { void *run_parse_feature(void *v) {
struct queue_run_arg *qra = (struct queue_run_arg *) v; struct queue_run_arg *qra = (struct queue_run_arg *) v;
key_pool key_pool;
for (size_t i = qra->start; i < qra->end; i++) { for (size_t i = qra->start; i < qra->end; i++) {
struct queued_feature &qf = feature_queue[i]; struct queued_feature &qf = feature_queue[i];
readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &(*qf.sst)[qra->segment], qf.layer, qf.layername); readFeature(qf.pbf, qf.dim, qf.e, *qf.keys, &(*qf.sst)[qra->segment], qf.layer, qf.layername, key_pool);
} }
return NULL; return NULL;
geocsv.cpp
+3 -2
View File
@@ -58,6 +58,7 @@ void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fnam
} }
size_t seq = 0; size_t seq = 0;
key_pool key_pool;
while ((s = csv_getline(f)).size() > 0) { while ((s = csv_getline(f)).size() > 0) {
std::string err = check_utf8(s); std::string err = check_utf8(s);
if (err != "") { if (err != "") {
@@ -89,7 +90,7 @@ void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fnam
drawvec dv; drawvec dv;
dv.push_back(draw(VT_MOVETO, x, y)); dv.push_back(draw(VT_MOVETO, x, y));
std::vector<std::string> full_keys; std::vector<std::shared_ptr<std::string>> full_keys;
std::vector<serial_val> full_values; std::vector<serial_val> full_values;
for (size_t i = 0; i < line.size(); i++) { for (size_t i = 0; i < line.size(); i++) {
@@ -107,7 +108,7 @@ void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fnam
} }
sv.s = line[i]; sv.s = line[i];
full_keys.push_back(header[i]); full_keys.push_back(key_pool.pool(header[i]));
full_values.push_back(sv); full_values.push_back(sv);
} }
} }
geojson.cpp
+5 -4
View File
@@ -182,17 +182,18 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
nprop = properties->value.object.length; nprop = properties->value.object.length;
} }
std::vector<std::string> keys; std::vector<std::shared_ptr<std::string>> full_keys;
std::vector<serial_val> values; std::vector<serial_val> values;
keys.reserve(nprop); full_keys.reserve(nprop);
values.reserve(nprop); values.reserve(nprop);
key_pool key_pool;
for (size_t i = 0; i < nprop; i++) { for (size_t i = 0; i < nprop; i++) {
if (properties->value.object.keys[i]->type == JSON_STRING) { if (properties->value.object.keys[i]->type == JSON_STRING) {
serial_val sv = stringify_value(properties->value.object.values[i], sst->fname, sst->line, feature); serial_val sv = stringify_value(properties->value.object.values[i], sst->fname, sst->line, feature);
keys.emplace_back(properties->value.object.keys[i]->value.string.string); full_keys.emplace_back(key_pool.pool(properties->value.object.keys[i]->value.string.string));
values.push_back(std::move(sv)); values.push_back(std::move(sv));
} }
} }
@@ -211,7 +212,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
sf.geometry = dv; sf.geometry = dv;
sf.feature_minzoom = 0; // Will be filled in during index merging sf.feature_minzoom = 0; // Will be filled in during index merging
sf.seq = *(sst->layer_seq); sf.seq = *(sst->layer_seq);
sf.full_keys = std::move(keys); sf.full_keys = std::move(full_keys);
sf.full_values = std::move(values); sf.full_values = std::move(values);
return serialize_feature(sst, sf, tippecanoe_layername); return serialize_feature(sst, sf, tippecanoe_layername);
plugin.cpp
+2 -2
View File
@@ -145,7 +145,7 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
} }
// Reads from the prefilter // Reads from the prefilter
serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool postfilter) { serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool postfilter, key_pool &key_pool) {
serial_feature sf; serial_feature sf;
while (1) { while (1) {
@@ -354,7 +354,7 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
// would have already run before prefiltering // would have already run before prefiltering
if (v.type != mvt_null) { if (v.type != mvt_null) {
sf.full_keys.push_back(std::string(properties->value.object.keys[i]->value.string.string)); sf.full_keys.push_back(key_pool.pool(std::string(properties->value.object.keys[i]->value.string.string)));
sf.full_values.push_back(v); sf.full_values.push_back(v);
if (!postfilter) { if (!postfilter) {
plugin.hpp
+2 -1
View File
@@ -1,3 +1,4 @@
struct key_pool;
std::vector<mvt_layer> filter_layers(const char *filter, std::vector<mvt_layer> &layer, unsigned z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, int extent); std::vector<mvt_layer> filter_layers(const char *filter, std::vector<mvt_layer> &layer, unsigned z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, int extent);
void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y); void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y);
serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool filters); serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool filters, key_pool &key_pool);
serial.cpp
+2 -1
View File
@@ -413,8 +413,9 @@ static void add_scaled_node(struct reader *r, serialization_state *sst, draw g)
} }
// called from frontends // called from frontends
int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername, key_pool &key_pool) { int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername) {
struct reader *r = &(*sst->readers)[sst->segment]; struct reader *r = &(*sst->readers)[sst->segment];
key_pool key_pool;
sf.bbox[0] = LLONG_MAX; sf.bbox[0] = LLONG_MAX;
sf.bbox[1] = LLONG_MAX; sf.bbox[1] = LLONG_MAX;
tile.cpp
+2 -1
View File
@@ -1063,6 +1063,7 @@ struct next_feature_state {
// geometry type set to -2. // geometry type set to -2.
static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, std::atomic<bool> *within, compressor **geomfile, std::atomic<long long> *geompos, long long start_geompos[], std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr<std::string> &tile_stringpool, std::vector<std::string> const &unidecode_data, next_feature_state &next_feature_state, double droprate) { static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, std::atomic<bool> *within, compressor **geomfile, std::atomic<long long> *geompos, long long start_geompos[], std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr<std::string> &tile_stringpool, std::vector<std::string> const &unidecode_data, next_feature_state &next_feature_state, double droprate) {
double extra_multiplier_zooms = log(retain_points_multiplier) / log(droprate); double extra_multiplier_zooms = log(retain_points_multiplier) / log(droprate);
while (1) { while (1) {
serial_feature sf; serial_feature sf;
long long len; long long len;
@@ -1859,7 +1860,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (prefilter == NULL) { if (prefilter == NULL) {
sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, start_geompos, &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, compressed_input, &multiplier_state, tile_stringpool, unidecode_data, next_feature_state, arg->droprate); sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, start_geompos, &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, compressed_input, &multiplier_state, tile_stringpool, unidecode_data, next_feature_state, arg->droprate);
} else { } else {
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL); sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL, key_pool);
} }
if (sf.t < 0) { if (sf.t < 0) {