diff --git a/attribute.cpp b/attribute.cpp index e582dc2a..2feb224d 100644 --- a/attribute.cpp +++ b/attribute.cpp @@ -89,9 +89,9 @@ void set_attribute_accum(std::unordered_map &attribut } template -static void preserve_attribute1(attribute_op const &op, std::string const &key, T const &val, std::vector &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state) { +static void preserve_attribute1(attribute_op const &op, std::string const &key, T const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool) { for (size_t i = 0; i < full_keys.size(); i++) { - if (key == full_keys[i]) { + if (key == *full_keys[i]) { switch (op) { case op_sum: full_values[i] = (full_values[i].to_double() + val.to_double()); @@ -193,14 +193,14 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key, exit(EXIT_IMPOSSIBLE); } - full_keys.push_back(key); + full_keys.push_back(key_pool.pool(key)); full_values.push_back(v); } -void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state) { - preserve_attribute1(op, key, val, full_keys, full_values, attribute_accum_state); +void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool) { + preserve_attribute1(op, key, val, full_keys, full_values, attribute_accum_state, key_pool); } -void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state) { - preserve_attribute1(op, key, val, full_keys, full_values, attribute_accum_state); +void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool) { + preserve_attribute1(op, key, val, full_keys, full_values, attribute_accum_state, key_pool); } diff --git a/attribute.hpp b/attribute.hpp index 84a76cfa..5dd9f2a2 100644 --- a/attribute.hpp +++ b/attribute.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "mvt.hpp" #include "milo/dtoa_milo.h" @@ -24,12 +25,13 @@ struct accum_state { }; struct serial_val; +struct key_pool; void set_attribute_accum(std::unordered_map &attribute_accum, std::string name, std::string type); void set_attribute_accum(std::unordered_map &attribute_accum, const char *arg, char **argv); -void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state); -void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state); +void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool); +void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool); extern std::map numeric_operations; diff --git a/clip.cpp b/clip.cpp index 6ecc775e..470b7538 100644 --- a/clip.cpp +++ b/clip.cpp @@ -1163,14 +1163,14 @@ static void add_mean(mvt_feature &feature, mvt_layer &layer, std::string const & }; // accumulate :sum:, :min:, :max:, and :count: versions of the specified attribute -static void preserve_numeric(const std::string &key, const mvt_value &val, // numeric attribute being accumulated - std::vector &full_keys, // keys of feature being accumulated onto - std::vector &full_values, // values of features being accumulated onto - const std::string &accumulate_numeric, // prefix of accumulations - std::set &keys, // key presence in the source feature - std::map &numeric_out_field, // key index in the output feature - std::unordered_map &attribute_accum_state // accumulation state for preserve_attribute() -) { +static void preserve_numeric(const std::string &key, const mvt_value &val, // numeric attribute being accumulated + std::vector> &full_keys, // keys of feature being accumulated onto + std::vector &full_values, // values of features being accumulated onto + const std::string &accumulate_numeric, // prefix of accumulations + std::set &keys, // key presence in the source feature + std::map &numeric_out_field, // key index in the output feature + std::unordered_map &attribute_accum_state, // accumulation state for preserve_attribute() + key_pool &key_pool) { // If this is a numeric attribute, but there is also a prefix:sum (etc.) for the // same attribute, we want to use that one instead of this one. @@ -1213,7 +1213,7 @@ static void preserve_numeric(const std::string &key, const mvt_value &val, / if (out_attr == numeric_out_field.end()) { // not present at all, so copy our value to the prefixed output numeric_out_field.emplace(prefixed, full_keys.size()); - full_keys.push_back(prefixed); + full_keys.push_back(key_pool.pool(prefixed)); if (op.second == op_count) { if (starting_from_accumulation) { @@ -1229,7 +1229,7 @@ static void preserve_numeric(const std::string &key, const mvt_value &val, / } else { // exists unprefixed, so copy it, and then accumulate on our value numeric_out_field.emplace(prefixed, full_keys.size()); - full_keys.push_back(prefixed); + full_keys.push_back(key_pool.pool(prefixed)); if (op.second == op_count) { mvt_value v; @@ -1243,7 +1243,7 @@ static void preserve_numeric(const std::string &key, const mvt_value &val, / full_values.push_back(v); } else { full_values.push_back(full_values[out_attr->second]); - preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state); + preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state, key_pool); } } } else { @@ -1256,7 +1256,7 @@ static void preserve_numeric(const std::string &key, const mvt_value &val, / full_values[prefixed_attr->second] = mvt_value(mvt_value_to_long_long(full_values[prefixed_attr->second]) + 1); } } else { - preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state); + preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state, key_pool); } } } @@ -1289,7 +1289,8 @@ static void feature_out(std::vector const &features, mvt_layer &ou std::set const &exclude, std::vector const &exclude_prefix, std::unordered_map const &attribute_accum, - std::string const &accumulate_numeric) { + std::string const &accumulate_numeric, + key_pool &key_pool) { // Add geometry to output feature mvt_feature outfeature; @@ -1315,7 +1316,7 @@ static void feature_out(std::vector const &features, mvt_layer &ou // multiplier cluster accumulated onto them std::unordered_map attribute_accum_state; - std::vector full_keys; + std::vector> full_keys; std::vector full_values; std::map numeric_out_field; @@ -1324,12 +1325,12 @@ static void feature_out(std::vector const &features, mvt_layer &ou auto f = attribute_accum.find(key); if (f != attribute_accum.end()) { // this attribute has an accumulator, so convert it - full_keys.push_back(features[0].layer->keys[features[0].tags[i]]); + full_keys.push_back(key_pool.pool(features[0].layer->keys[features[0].tags[i]])); full_values.push_back(features[0].layer->values[features[0].tags[i + 1]]); } else if (accumulate_numeric.size() > 0 && features[0].layer->values[features[0].tags[i + 1]].is_numeric()) { // convert numeric for accumulation numeric_out_field.emplace(key, full_keys.size()); - full_keys.push_back(key); + full_keys.push_back(key_pool.pool(key)); full_values.push_back(features[0].layer->values[features[0].tags[i + 1]]); } else { // otherwise just tag it directly onto the output feature @@ -1357,13 +1358,13 @@ static void feature_out(std::vector const &features, mvt_layer &ou auto f = attribute_accum.find(key); if (f != attribute_accum.end()) { mvt_value val = features[i].layer->values[features[i].tags[j + 1]]; - preserve_attribute(f->second, key, val, full_keys, full_values, attribute_accum_state); + preserve_attribute(f->second, key, val, full_keys, full_values, attribute_accum_state, key_pool); } else if (accumulate_numeric.size() > 0) { const mvt_value &val = features[i].layer->values[features[i].tags[j + 1]]; if (val.is_numeric()) { preserve_numeric(key, val, full_keys, full_values, accumulate_numeric, - keys, numeric_out_field, attribute_accum_state); + keys, numeric_out_field, attribute_accum_state, key_pool); } } } @@ -1373,8 +1374,8 @@ static void feature_out(std::vector const &features, mvt_layer &ou // and tag them onto the output feature for (size_t i = 0; i < full_keys.size(); i++) { - if (should_keep(full_keys[i], keep, exclude, exclude_prefix)) { - outlayer.tag(outfeature, full_keys[i], full_values[i]); + if (should_keep(*full_keys[i], keep, exclude, exclude_prefix)) { + outlayer.tag(outfeature, *full_keys[i], full_values[i]); } } @@ -1522,6 +1523,7 @@ mvt_tile assign_to_bins(mvt_tile &features, std::set exclude, std::vector exclude_prefix) { std::vector events; + key_pool key_pool; // Index bins for (size_t i = 0; i < bins.size(); i++) { @@ -1678,7 +1680,7 @@ mvt_tile assign_to_bins(mvt_tile &features, if (outfeatures[i].size() > 1) { feature_out(outfeatures[i], outlayer, keep, exclude, exclude_prefix, attribute_accum, - accumulate_numeric); + accumulate_numeric, key_pool); mvt_feature &nfeature = outlayer.features.back(); mvt_value val; val.type = mvt_uint; @@ -1713,6 +1715,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int std::vector const &bins, std::string const &bin_by_id_list, std::string const &accumulate_numeric) { mvt_tile outtile; + key_pool key_pool; for (auto const &tile : tiles) { for (auto const &layer : tile.tile.layers) { @@ -1837,7 +1840,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int if (flush_multiplier_cluster) { if (pending_tile_features.size() > 0) { - feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric); + feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool); pending_tile_features.clear(); } } @@ -1894,7 +1897,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int } if (pending_tile_features.size() > 0) { - feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric); + feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool); pending_tile_features.clear(); } diff --git a/flatgeobuf.cpp b/flatgeobuf.cpp index cf72d671..571bb8fb 100644 --- a/flatgeobuf.cpp +++ b/flatgeobuf.cpp @@ -146,8 +146,9 @@ void readFeature(const FlatGeobuf::Feature *feature, long long feature_sequence_ sf.geometry = dv; sf.t = drawvec_type; - std::vector full_keys; + std::vector> full_keys; std::vector full_values; + key_pool key_pool; // assume tabular schema with columns in header size_t p_pos = 0; @@ -243,7 +244,7 @@ void readFeature(const FlatGeobuf::Feature *feature, long long feature_sequence_ fprintf(stderr, "flatgeobuf has unsupported column type %u\n", (unsigned int)col_type); exit(EXIT_IMPOSSIBLE); } - full_keys.push_back(h_column_names[col_idx]); + full_keys.push_back(key_pool.pool(h_column_names[col_idx])); full_values.push_back(sv); } diff --git a/serial.cpp b/serial.cpp index 3b356c89..448fcea6 100644 --- a/serial.cpp +++ b/serial.cpp @@ -413,7 +413,7 @@ static void add_scaled_node(struct reader *r, serialization_state *sst, draw g) } // called from frontends -int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername) { +int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername, key_pool &key_pool) { struct reader *r = &(*sst->readers)[sst->segment]; sf.bbox[0] = LLONG_MAX; @@ -714,7 +714,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: bbox_index = encode_index(midx, midy); if (additional[A_CALCULATE_INDEX]) { - sf.full_keys.push_back("tippecanoe:index"); + sf.full_keys.push_back(key_pool.pool("tippecanoe:index")); serial_val sv; sv.type = mvt_double; @@ -776,7 +776,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: for (auto &kv : set_attributes) { bool found = false; for (size_t i = 0; i < sf.full_keys.size(); i++) { - if (sf.full_keys[i] == kv.first) { + if (*sf.full_keys[i] == kv.first) { sf.full_values[i] = kv.second; found = true; break; @@ -784,13 +784,13 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: } if (!found) { - sf.full_keys.push_back(kv.first); + sf.full_keys.push_back(key_pool.pool(kv.first)); sf.full_values.push_back(kv.second); } } for (ssize_t i = (ssize_t) sf.full_keys.size() - 1; i >= 0; i--) { - coerce_value(sf.full_keys[i], sf.full_values[i].type, sf.full_values[i].s, sst->attribute_types); + coerce_value(*sf.full_keys[i], sf.full_values[i].type, sf.full_values[i].s, sst->attribute_types); if (prevent[P_SINGLE_PRECISION]) { if (sf.full_values[i].type == mvt_double) { @@ -801,12 +801,12 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: } } - if (sf.full_keys[i] == attribute_for_id) { + if (*sf.full_keys[i] == attribute_for_id) { if (sf.full_values[i].type != mvt_double && !additional[A_CONVERT_NUMERIC_IDS]) { static bool warned = false; if (!warned) { - fprintf(stderr, "Warning: Attribute \"%s\"=\"%s\" as feature ID is not a number\n", sf.full_keys[i].c_str(), sf.full_values[i].s.c_str()); + fprintf(stderr, "Warning: Attribute \"%s\"=\"%s\" as feature ID is not a number\n", sf.full_keys[i]->c_str(), sf.full_values[i].s.c_str()); warned = true; } } else { @@ -839,12 +839,12 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: } if (sst->exclude_all) { - if (sst->include->count(sf.full_keys[i]) == 0) { + if (sst->include->count(*sf.full_keys[i]) == 0) { sf.full_keys.erase(sf.full_keys.begin() + i); sf.full_values.erase(sf.full_values.begin() + i); continue; } - } else if (sst->exclude->count(sf.full_keys[i]) != 0) { + } else if (sst->exclude->count(*sf.full_keys[i]) != 0) { sf.full_keys.erase(sf.full_keys.begin() + i); sf.full_values.erase(sf.full_values.begin() + i); continue; @@ -854,7 +854,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: if (!sst->filters) { for (size_t i = 0; i < sf.full_keys.size(); i++) { auto ts = sst->layermap->find(layername); - add_to_tilestats(ts->second.tilestats, sf.full_keys[i], sf.full_values[i]); + add_to_tilestats(ts->second.tilestats, *sf.full_keys[i], sf.full_values[i]); } } @@ -867,7 +867,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: } for (size_t i = 0; i < sf.full_keys.size(); i++) { - sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string, r->key_dedup)); + sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i]->c_str(), mvt_string, r->key_dedup)); sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type, r->value_dedup)); } diff --git a/serial.hpp b/serial.hpp index 80b67ce0..91cdac5c 100644 --- a/serial.hpp +++ b/serial.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "geometry.hpp" #include "mbtiles.hpp" @@ -71,6 +72,22 @@ struct serial_val { } }; +struct key_pool { + std::unordered_map> mapping; + + std::shared_ptr pool(std::string const &s) { + auto f = mapping.find(s); + if (f != mapping.end()) { + return f->second; + } + + std::shared_ptr p = std::make_shared(); + *p = s; + mapping.emplace(s, p); + return p; + } +}; + struct serial_feature { long long layer = 0; int segment = 0; @@ -95,7 +112,7 @@ struct serial_feature { // to create the keys and values references into the string pool // during initial serialization - std::vector full_keys{}; + std::vector> full_keys{}; std::vector full_values{}; // These fields are generated from full_keys and full_values diff --git a/tile.cpp b/tile.cpp index 153c8166..74ec407b 100644 --- a/tile.cpp +++ b/tile.cpp @@ -143,9 +143,9 @@ static int coalcmp(const void *v1, const void *v2) { } for (size_t i = 0; i < c1->full_keys.size(); i++) { - if (c1->full_keys[i] < c2->full_keys[i]) { + if (*c1->full_keys[i] < *c2->full_keys[i]) { return -1; - } else if (c1->full_keys[i] > c2->full_keys[i]) { + } else if (*c1->full_keys[i] > *c2->full_keys[i]) { return 1; } @@ -302,7 +302,7 @@ static mvt_value find_attribute_value(const serial_feature *c1, std::string cons } for (size_t i = 0; i < c1->full_keys.size(); i++) { - if (c1->full_keys[i] == key) { + if (*c1->full_keys[i] == key) { return stringified_to_mvt_value(c1->full_values[i].type, c1->full_values[i].s.c_str(), c1->tile_stringpool); } } @@ -384,7 +384,7 @@ static std::vector> assemble_multiplier_clusters(std bool is_cluster_start = false; for (size_t i = 0; i < feature.full_keys.size(); i++) { - if (feature.full_keys[i] == "tippecanoe:retain_points_multiplier_first") { + if (*feature.full_keys[i] == "tippecanoe:retain_points_multiplier_first") { is_cluster_start = true; break; } @@ -412,7 +412,7 @@ static std::vector disassemble_multiplier_clusters(std::vector const &e } for (ssize_t i = sf.full_keys.size() - 1; i >= 0; i--) { - std::string key = sf.full_keys[i]; + std::string key = *sf.full_keys[i]; if (exclude_attributes.count(key) > 0) { sf.full_keys.erase(sf.full_keys.begin() + i); sf.full_values.erase(sf.full_values.begin() + i); @@ -1183,7 +1183,7 @@ static serial_feature next_feature(decompressor *geoms, std::atomic * } for (size_t i = 0; i < sf.full_keys.size(); i++) { - std::string key = sf.full_keys[i]; + std::string key = *sf.full_keys[i]; mvt_value val = stringified_to_mvt_value(sf.full_values[i].type, sf.full_values[i].s.c_str(), tile_stringpool); attributes.insert(std::pair(key, val)); @@ -1399,7 +1399,7 @@ void add_tilestats(std::string const &layername, int z, std::vectorsecond.tilestats, key, val); } -void promote_attribute(std::string const &key, serial_feature &p) { +void promote_attribute(std::string const &key, serial_feature &p, key_pool &key_pool) { if (p.need_tilestats.count(key) == 0) { p.need_tilestats.insert(key); } @@ -1413,7 +1413,7 @@ void promote_attribute(std::string const &key, serial_feature &p) { sv.s = p.stringpool + p.values[i] + 1; sv.type = p.stringpool[p.values[i]]; - p.full_keys.push_back(key); + p.full_keys.push_back(key_pool.pool(key)); p.full_values.push_back(std::move(sv)); p.keys.erase(p.keys.begin() + i); @@ -1424,7 +1424,7 @@ void promote_attribute(std::string const &key, serial_feature &p) { } } -void promote_attribute_prefix(std::string const &key, std::string const &prefixed_key, serial_feature &p) { +void promote_attribute_prefix(std::string const &key, std::string const &prefixed_key, serial_feature &p, key_pool &key_pool) { if (p.need_tilestats.count(prefixed_key) == 0) { p.need_tilestats.insert(prefixed_key); } @@ -1432,18 +1432,18 @@ void promote_attribute_prefix(std::string const &key, std::string const &prefixe // does the prefixed attribute already exist as a full key? ssize_t found_as = -1; for (size_t i = 0; i < p.full_keys.size(); i++) { - if (prefixed_key == p.full_keys[i]) { + if (prefixed_key == *p.full_keys[i]) { // yes, so we're done return; } - if (key == p.full_keys[i]) { + if (key == *p.full_keys[i]) { found_as = i; } } // or did we find the source as a full key? then copy it if (found_as >= 0) { - p.full_keys.push_back(prefixed_key); + p.full_keys.push_back(key_pool.pool(prefixed_key)); p.full_values.push_back(p.full_values[found_as]); return; } @@ -1457,7 +1457,7 @@ void promote_attribute_prefix(std::string const &key, std::string const &prefixe sv.s = p.stringpool + p.values[i] + 1; sv.type = p.stringpool[p.values[i]]; - p.full_keys.push_back(prefixed_key); + p.full_keys.push_back(key_pool.pool(prefixed_key)); p.full_values.push_back(std::move(sv)); p.keys.erase(p.keys.begin() + i); @@ -1475,7 +1475,7 @@ void promote_attribute_prefix(std::string const &key, std::string const &prefixe sv.s = p.stringpool + p.values[found_as] + 1; sv.type = p.stringpool[p.values[found_as]]; - p.full_keys.push_back(prefixed_key); + p.full_keys.push_back(key_pool.pool(prefixed_key)); p.full_values.push_back(std::move(sv)); return; @@ -1485,7 +1485,7 @@ void promote_attribute_prefix(std::string const &key, std::string const &prefixe } // accumulate attribute values from sf onto p -void preserve_attributes(std::unordered_map const *attribute_accum, const serial_feature &sf, serial_feature &p) { +void preserve_attributes(std::unordered_map const *attribute_accum, const serial_feature &sf, serial_feature &p, key_pool &key_pool) { std::string accumulate_numeric_colon = accumulate_numeric + ":"; for (size_t i = 0; i < sf.keys.size(); i++) { @@ -1498,8 +1498,8 @@ void preserve_attributes(std::unordered_map const *at sv.type = sf.stringpool[sf.values[i]]; sv.s = sf.stringpool + sf.values[i] + 1; - promote_attribute(key, p); - preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state); + promote_attribute(key, p, key_pool); + preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state, key_pool); } else if (type == mvt_double && accumulate_numeric.size() > 0 && !starts_with(key, accumulate_numeric_colon)) { for (auto const &operation : numeric_operations) { serial_val sv; @@ -1507,26 +1507,26 @@ void preserve_attributes(std::unordered_map const *at sv.s = sf.stringpool + sf.values[i] + 1; std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key; - promote_attribute_prefix(key, prefixed_key, p); - preserve_attribute(operation.second, prefixed_key, sv, p.full_keys, p.full_values, p.attribute_accum_state); + promote_attribute_prefix(key, prefixed_key, p, key_pool); + preserve_attribute(operation.second, prefixed_key, sv, p.full_keys, p.full_values, p.attribute_accum_state, key_pool); } } } for (size_t i = 0; i < sf.full_keys.size(); i++) { - const std::string &key = sf.full_keys[i]; + const std::string key = *sf.full_keys[i]; int type = sf.full_values[i].type; auto f = attribute_accum->find(key); if (f != attribute_accum->end()) { const serial_val &sv = sf.full_values[i]; - promote_attribute(key, p); // promotes it in the target feature - preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state); + promote_attribute(key, p, key_pool); // promotes it in the target feature + preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state, key_pool); } else if (type == mvt_double && accumulate_numeric.size() > 0 && !starts_with(key, accumulate_numeric_colon)) { for (auto const &operation : numeric_operations) { std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key; - promote_attribute_prefix(key, prefixed_key, p); - preserve_attribute(operation.second, prefixed_key, sf.full_values[i], p.full_keys, p.full_values, p.attribute_accum_state); + promote_attribute_prefix(key, prefixed_key, p, key_pool); + preserve_attribute(operation.second, prefixed_key, sf.full_values[i], p.full_keys, p.full_values, p.attribute_accum_state, key_pool); } } } @@ -1620,7 +1620,7 @@ struct layer_features { size_t multiplier_cluster_size = 0; // The feature count of the current multiplier cluster }; -bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features &layer, serial_feature &sf, std::vector> *layer_unmaps, strategy &strategy, bool &drop_rest, std::unordered_map const *attribute_accum) { +bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features &layer, serial_feature &sf, std::vector> *layer_unmaps, strategy &strategy, bool &drop_rest, std::unordered_map const *attribute_accum, key_pool &key_pool) { ssize_t which_serial_feature; if (find_feature_to_accumulate_onto(layer.features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { @@ -1631,7 +1631,7 @@ bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features sf.dropped = layer.multiplier_cluster_size + 1; return false; // converted rather than dropped } else { - preserve_attributes(attribute_accum, sf, layer.features[which_serial_feature]); + preserve_attributes(attribute_accum, sf, layer.features[which_serial_feature], key_pool); drop_rest = true; return true; // dropped } @@ -1741,6 +1741,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch bool too_many_features = false; bool too_many_bytes = false; + key_pool key_pool; + std::atomic within[child_shards]; long long start_geompos[child_shards]; for (size_t i = 0; i < (size_t) child_shards; i++) { @@ -1918,7 +1920,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (sf.dropped == FEATURE_DROPPED || drop_rest) { if (find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); + preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); strategy.dropped_by_rate++; can_stop_early = false; continue; @@ -1931,7 +1933,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (sf.dropped == FEATURE_KEPT) { if (gamma > 0) { if (manage_gap(sf.index, &previndex, scale, gamma, &gap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); + preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); strategy.dropped_by_gamma++; drop_rest = true; can_stop_early = false; @@ -1959,7 +1961,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch features[which_serial_feature].geometry[0].y = y / (features[which_serial_feature].clustered + 1); } - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); + preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; @@ -1969,7 +1971,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch add_sample_to(gaps, sf.gap, gaps_increment, seq); if (sf.gap < mingap) { can_stop_early = false; - if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum)) { + if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum, key_pool)) { continue; } } @@ -1991,7 +1993,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch features[which_serial_feature].geometry[0].y = y / (features[which_serial_feature].clustered + 1); } - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); + preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; continue; @@ -2002,7 +2004,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch coalesce_geometry(features[which_serial_feature], sf); features[which_serial_feature].coalesced = true; coalesced_area += sf.extent; - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); + preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; @@ -2014,7 +2016,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch // so we shouldn't expect to find anything small that we can related this feature to. if (minextent != 0 && sf.extent + coalesced_area <= minextent) { can_stop_early = false; - if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum)) { + if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum, key_pool)) { continue; } } @@ -2024,7 +2026,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch coalesce_geometry(features[which_serial_feature], sf); features[which_serial_feature].coalesced = true; coalesced_area += sf.extent; - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); + preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; @@ -2034,7 +2036,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch add_sample_to(drop_sequences, drop_sequence, drop_sequences_increment, seq); if (mindrop_sequence != 0 && drop_sequence <= mindrop_sequence) { can_stop_early = false; - if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum)) { + if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum, key_pool)) { continue; } } @@ -2043,7 +2045,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (mindrop_sequence != 0 && drop_sequence <= mindrop_sequence && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { coalesce_geometry(features[which_serial_feature], sf); features[which_serial_feature].coalesced = true; - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]); + preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; @@ -2143,7 +2145,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } if (sf.dropped == FEATURE_KEPT && retain_points_multiplier > 1) { - sf.full_keys.push_back("tippecanoe:retain_points_multiplier_first"); + sf.full_keys.push_back(key_pool.pool("tippecanoe:retain_points_multiplier_first")); sf.full_values.emplace_back(mvt_bool, "true"); } @@ -2313,10 +2315,10 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch size_t j = feature_sequences[i].second; serial_val sv(mvt_double, std::to_string(i)); - features[j].full_keys.push_back("tippecanoe:retain_points_multiplier_sequence"); + features[j].full_keys.push_back(key_pool.pool("tippecanoe:retain_points_multiplier_sequence")); features[j].full_values.push_back(sv); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, features[j].full_keys.back(), sv); + add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *features[j].full_keys.back(), sv); } } @@ -2328,28 +2330,28 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch long long point_count = p.clustered + 1; char abbrev[20]; // to_string(LLONG_MAX).length() / 1000 + 1; - p.full_keys.push_back("clustered"); + p.full_keys.push_back(key_pool.pool("clustered")); sv.type = mvt_bool; sv.s = "true"; p.full_values.push_back(sv); add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "clustered", sv); - p.full_keys.push_back("point_count"); + p.full_keys.push_back(key_pool.pool("point_count")); sv2.type = mvt_double; sv2.s = std::to_string(point_count); p.full_values.push_back(sv2); add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "point_count", sv2); - p.full_keys.push_back("sqrt_point_count"); + p.full_keys.push_back(key_pool.pool("sqrt_point_count")); sv3.type = mvt_double; sv3.s = std::to_string(round(100 * sqrt(point_count)) / 100.0); p.full_values.push_back(sv3); add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "sqrt_point_count", sv3); - p.full_keys.push_back("point_count_abbreviated"); + p.full_keys.push_back(key_pool.pool("point_count_abbreviated")); sv4.type = mvt_string; if (point_count >= 10000) { snprintf(abbrev, sizeof(abbrev), "%.0fk", point_count / 1000.0); @@ -2366,8 +2368,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (p.need_tilestats.size() > 0) { for (size_t j = 0; j < p.full_keys.size(); j++) { - if (p.need_tilestats.count(p.full_keys[j]) > 0) { - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, p.full_keys[j], p.full_values[j]); + if (p.need_tilestats.count(*p.full_keys[j]) > 0) { + add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]); } } } @@ -2559,7 +2561,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch for (size_t a = 0; a < layer_features[x].full_keys.size(); a++) { serial_val sv = layer_features[x].full_values[a]; mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); - layer.tag(feature, layer_features[x].full_keys[a], v); + layer.tag(feature, *layer_features[x].full_keys[a], v); } if (additional[A_CALCULATE_FEATURE_DENSITY]) {