From 583fc3744a43ca963788ab9728123a6e64dc7ef0 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Thu, 30 Jan 2025 21:58:35 -0800 Subject: [PATCH] Reduce attribute accumulation memory consumption (#318) * Add a flag to use an H3 index for the feature index * Give mvt_value and serial_val a double-with-count concept * Switch mean over to internal accumulation state * Get rid of the attribute accumulation map * Change vectors of features to vectors of pointers to features * Fix --coalesce * Revert "Add a flag to use an H3 index for the feature index" This reverts commit b9b48f42c98c5a277fde5cb49eed28330ee5504e. * Update version and changelog --- CHANGELOG.md | 4 + attribute.cpp | 55 ++-- attribute.hpp | 9 +- clip.cpp | 22 +- mvt.hpp | 15 +- serial.hpp | 15 +- shared_borders.cpp | 94 +++---- shared_borders.hpp | 2 +- ...theconcat%3aconcat_-Ethecomma%3acomma.json | 4 +- tile.cpp | 253 +++++++++--------- version.hpp | 2 +- 11 files changed, 243 insertions(+), 232 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78bf5b53..095a17fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 2.75.0 + +* Reduce memory consumption in attribute accumulation and feature sorting + # 2.74.0 * Add the option to join attributes from a sqlite database in tile-join diff --git a/attribute.cpp b/attribute.cpp index 2feb224d..7f33ade9 100644 --- a/attribute.cpp +++ b/attribute.cpp @@ -89,7 +89,7 @@ void set_attribute_accum(std::unordered_map &attribut } template -static void preserve_attribute1(attribute_op const &op, std::string const &key, T const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool) { +static void preserve_attribute1(attribute_op const &op, std::string const &key, T const &val, std::vector> &full_keys, std::vector &full_values, key_pool &key_pool) { for (size_t i = 0; i < full_keys.size(); i++) { if (key == *full_keys[i]) { switch (op) { @@ -120,19 +120,13 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key, } case op_mean: { - auto state = attribute_accum_state.find(key); - if (state == attribute_accum_state.end()) { - accum_state s; - s.sum = full_values[i].to_double() + val.to_double(); - s.count = 2; - attribute_accum_state.insert(std::pair(key, s)); - - full_values[i] = (s.sum / s.count); + size_t count = full_values[i].get_count(); + if (count <= 1) { + full_values[i].set_double_count((full_values[i].to_double() + val.to_double()) / 2, 2); } else { - state->second.sum += val.to_double(); - state->second.count += 1; - - full_values[i] = (state->second.sum / state->second.count); + double sum = full_values[i].to_double() * count + val.to_double(); + count++; + full_values[i].set_double_count(sum / count, count); } return; } @@ -146,16 +140,11 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key, return; case op_count: { - auto state = attribute_accum_state.find(key); - if (state == attribute_accum_state.end()) { // not already present - accum_state s; - s.count = 2; - attribute_accum_state.insert(std::pair(key, s)); - - full_values[i] = (s.count); - } else { // already present, incrementing - state->second.count += 1; - full_values[i] = (state->second.count); + size_t count = full_values[i].get_count(); + if (count <= 1) { + full_values[i].set_double_count(2, 2); + } else { + full_values[i].set_double_count(count + 1, count + 1); } return; } @@ -174,17 +163,7 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key, break; case op_count: { - auto state = attribute_accum_state.find(key); - if (state == attribute_accum_state.end()) { // not already present - accum_state s; - s.count = 1; - attribute_accum_state.insert(std::pair(key, s)); - - v = (s.count); - } else { // already present, incrementing - fprintf(stderr, "preserve_attribute: can't happen (count)\n"); - exit(EXIT_IMPOSSIBLE); - } + v.set_double_count(1, 1); break; } @@ -197,10 +176,10 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key, full_values.push_back(v); } -void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool) { - preserve_attribute1(op, key, val, full_keys, full_values, attribute_accum_state, key_pool); +void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector> &full_keys, std::vector &full_values, key_pool &key_pool) { + preserve_attribute1(op, key, val, full_keys, full_values, key_pool); } -void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool) { - preserve_attribute1(op, key, val, full_keys, full_values, attribute_accum_state, key_pool); +void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector> &full_keys, std::vector &full_values, key_pool &key_pool) { + preserve_attribute1(op, key, val, full_keys, full_values, key_pool); } diff --git a/attribute.hpp b/attribute.hpp index 5dd9f2a2..f66de8c1 100644 --- a/attribute.hpp +++ b/attribute.hpp @@ -19,19 +19,14 @@ enum attribute_op { op_count, }; -struct accum_state { - double sum = 0; - double count = 0; -}; - struct serial_val; struct key_pool; void set_attribute_accum(std::unordered_map &attribute_accum, std::string name, std::string type); void set_attribute_accum(std::unordered_map &attribute_accum, const char *arg, char **argv); -void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool); -void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector> &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state, key_pool &key_pool); +void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector> &full_keys, std::vector &full_values, key_pool &key_pool); +void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector> &full_keys, std::vector &full_values, key_pool &key_pool); extern std::map numeric_operations; diff --git a/clip.cpp b/clip.cpp index 76d23089..69decd45 100644 --- a/clip.cpp +++ b/clip.cpp @@ -1371,13 +1371,12 @@ static void add_mean(mvt_feature &feature, mvt_layer &layer, std::string const & }; // accumulate :sum:, :min:, :max:, and :count: versions of the specified attribute -static void preserve_numeric(const std::string &key, const mvt_value &val, // numeric attribute being accumulated - std::vector> &full_keys, // keys of feature being accumulated onto - std::vector &full_values, // values of features being accumulated onto - const std::string &accumulate_numeric, // prefix of accumulations - std::set &keys, // key presence in the source feature - std::map &numeric_out_field, // key index in the output feature - std::unordered_map &attribute_accum_state, // accumulation state for preserve_attribute() +static void preserve_numeric(const std::string &key, const mvt_value &val, // numeric attribute being accumulated + std::vector> &full_keys, // keys of feature being accumulated onto + std::vector &full_values, // values of features being accumulated onto + const std::string &accumulate_numeric, // prefix of accumulations + std::set &keys, // key presence in the source feature + std::map &numeric_out_field, // key index in the output feature key_pool &key_pool, std::set const &keep, std::set const &exclude, std::vector const &exclude_prefix) { @@ -1457,7 +1456,7 @@ static void preserve_numeric(const std::string &key, const mvt_value &val, full_values.push_back(v); } else { full_values.push_back(full_values[out_attr->second]); - preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state, key_pool); + preserve_attribute(op.second, prefixed, val, full_keys, full_values, key_pool); } } } else { @@ -1470,7 +1469,7 @@ static void preserve_numeric(const std::string &key, const mvt_value &val, full_values[prefixed_attr->second] = mvt_value(mvt_value_to_long_long(full_values[prefixed_attr->second]) + 1); } } else { - preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state, key_pool); + preserve_attribute(op.second, prefixed, val, full_keys, full_values, key_pool); } } } @@ -1597,7 +1596,6 @@ static bool feature_out(std::vector const &features, mvt_layer &ou // attributes from the other features of the // multiplier cluster accumulated onto them - std::unordered_map attribute_accum_state; std::vector> full_keys; std::vector full_values; std::map numeric_out_field; @@ -1642,13 +1640,13 @@ static bool feature_out(std::vector const &features, mvt_layer &ou auto found = attribute_accum.find(key); if (found != attribute_accum.end()) { mvt_value val = features[i].layer->values[features[i].tags[j + 1]]; - preserve_attribute(found->second, key, val, full_keys, full_values, attribute_accum_state, key_pool); + preserve_attribute(found->second, key, val, full_keys, full_values, key_pool); } else if (accumulate_numeric.size() > 0) { const mvt_value &val = features[i].layer->values[features[i].tags[j + 1]]; if (val.is_numeric()) { preserve_numeric(key, val, full_keys, full_values, accumulate_numeric, - keys, numeric_out_field, attribute_accum_state, key_pool, + keys, numeric_out_field, key_pool, keep, exclude, exclude_prefix); } } diff --git a/mvt.hpp b/mvt.hpp index e44a632a..8f24fc68 100644 --- a/mvt.hpp +++ b/mvt.hpp @@ -81,7 +81,8 @@ struct mvt_value; double mvt_value_to_double(mvt_value const &v); struct mvt_value { - mvt_value_type type; + long /* mvt_value_type */ type : 5; + long count : 64 - 5; std::shared_ptr s; union { @@ -138,6 +139,10 @@ struct mvt_value { return mvt_value_to_double(*this); } + size_t get_count() const { + return count; + } + bool operator<(const mvt_value &o) const; bool operator==(const mvt_value &o) const; std::string toString() const; @@ -145,11 +150,19 @@ struct mvt_value { mvt_value() { this->type = mvt_double; this->numeric_value.double_value = 0; + this->count = 0; } mvt_value(double v) { this->type = mvt_double; this->numeric_value.double_value = v; + this->count = 0; + } + + void set_double_count(double v, size_t c) { + this->type = mvt_double; + this->numeric_value.double_value = v; + this->count = c; } }; diff --git a/serial.hpp b/serial.hpp index 91cdac5c..1c86cc05 100644 --- a/serial.hpp +++ b/serial.hpp @@ -70,6 +70,20 @@ struct serial_val { type = mvt_string; s = val; } + + size_t get_count() const { + size_t found = s.find('\0'); + if (found == std::string::npos) { + return 0; + } else { + return atoll(s.c_str() + found + 1); + } + } + + void set_double_count(double v, size_t c) { + type = mvt_double; + s = milo::dtoa_milo(v) + '\0' + std::to_string(c); + } }; struct key_pool { @@ -152,7 +166,6 @@ struct serial_feature { const char *stringpool; // string pool for keys/values lookup std::shared_ptr tile_stringpool; // string pool for mvt_value construction std::set need_tilestats; - std::unordered_map attribute_accum_state; int z; // tile being produced int tx; diff --git a/shared_borders.cpp b/shared_borders.cpp index 709c8a21..c2496f9d 100644 --- a/shared_borders.cpp +++ b/shared_borders.cpp @@ -83,13 +83,13 @@ bool edges_same(std::pair::iterator, std::vector::iterat return true; } -bool find_common_edges(std::vector &features, int z, int line_detail, double simplification, int maxzoom, double merge_fraction) { +bool find_common_edges(std::vector> &features, int z, int line_detail, double simplification, int maxzoom, double merge_fraction) { size_t merge_count = ceil((1 - merge_fraction) * features.size()); for (size_t i = 0; i < features.size(); i++) { - if (features[i].t == VT_POLYGON) { + if (features[i]->t == VT_POLYGON) { { - drawvec &g = features[i].geometry; + drawvec &g = features[i]->geometry; drawvec out; for (size_t k = 0; k < g.size(); k++) { @@ -100,7 +100,7 @@ bool find_common_edges(std::vector &features, int z, int line_de } } - features[i].geometry = out; + features[i]->geometry = out; } } } @@ -112,21 +112,21 @@ bool find_common_edges(std::vector &features, int z, int line_de std::vector edges; size_t ring = 0; for (size_t i = 0; i < features.size(); i++) { - if (features[i].t == VT_POLYGON) { + if (features[i]->t == VT_POLYGON) { { - for (size_t k = 0; k + 1 < features[i].geometry.size(); k++) { - if (features[i].geometry[k].op == VT_MOVETO) { + for (size_t k = 0; k + 1 < features[i]->geometry.size(); k++) { + if (features[i]->geometry[k].op == VT_MOVETO) { ring++; } - if (features[i].geometry[k + 1].op == VT_LINETO) { + if (features[i]->geometry[k + 1].op == VT_LINETO) { drawvec dv; - if (features[i].geometry[k] < features[i].geometry[k + 1]) { - dv.push_back(features[i].geometry[k]); - dv.push_back(features[i].geometry[k + 1]); + if (features[i]->geometry[k] < features[i]->geometry[k + 1]) { + dv.push_back(features[i]->geometry[k]); + dv.push_back(features[i]->geometry[k + 1]); } else { - dv.push_back(features[i].geometry[k + 1]); - dv.push_back(features[i].geometry[k]); + dv.push_back(features[i]->geometry[k + 1]); + dv.push_back(features[i]->geometry[k]); } edges.push_back(edge(dv[0].x, dv[0].y, dv[1].x, dv[1].y, ring)); @@ -143,9 +143,9 @@ bool find_common_edges(std::vector &features, int z, int line_de // is not the same as the set of rings using the edge on the other side. for (size_t i = 0; i < features.size(); i++) { - if (features[i].t == VT_POLYGON) { + if (features[i]->t == VT_POLYGON) { { - drawvec &g = features[i].geometry; + drawvec &g = features[i]->geometry; for (size_t k = 0; k < g.size(); k++) { g[k].necessary = 0; @@ -223,9 +223,9 @@ bool find_common_edges(std::vector &features, int z, int line_de // Roll rings that include a necessary point around so they start at one for (size_t i = 0; i < features.size(); i++) { - if (features[i].t == VT_POLYGON) { + if (features[i]->t == VT_POLYGON) { { - drawvec &g = features[i].geometry; + drawvec &g = features[i]->geometry; for (size_t k = 0; k < g.size(); k++) { if (necessaries.count(g[k]) != 0) { @@ -315,21 +315,21 @@ bool find_common_edges(std::vector &features, int z, int line_de // Add new arc size_t added = arcs.size() + 1; arcs.insert(std::pair(arc, added)); - features[i].arc_polygon.push_back(added); + features[i]->arc_polygon.push_back(added); merge_candidates.insert(std::pair(added, i)); } else { - features[i].arc_polygon.push_back(-(ssize_t) f2->second); + features[i]->arc_polygon.push_back(-(ssize_t) f2->second); merge_candidates.insert(std::pair(-(ssize_t) f2->second, i)); } } else { - features[i].arc_polygon.push_back(f->second); + features[i]->arc_polygon.push_back(f->second); merge_candidates.insert(std::pair(f->second, i)); } m = n - 1; } - features[i].arc_polygon.push_back(0); + features[i]->arc_polygon.push_back(0); k = l - 1; } @@ -385,10 +385,10 @@ bool find_common_edges(std::vector &features, int z, int line_de if (r1i->second != r2i->second) { merge_order mo; mo.edge = i; - if (features[r1i->second].index > features[r2i->second].index) { - mo.gap = features[r1i->second].index - features[r2i->second].index; + if (features[r1i->second]->index > features[r2i->second]->index) { + mo.gap = features[r1i->second]->index - features[r2i->second]->index; } else { - mo.gap = features[r2i->second].index - features[r1i->second].index; + mo.gap = features[r2i->second]->index - features[r1i->second]->index; } mo.p1 = r1i->second; mo.p2 = r2i->second; @@ -406,19 +406,19 @@ bool find_common_edges(std::vector &features, int z, int line_de } size_t i = order[o].p1; - while (features[i].renamed >= 0) { - i = features[i].renamed; + while (features[i]->renamed >= 0) { + i = features[i]->renamed; } size_t i2 = order[o].p2; - while (features[i2].renamed >= 0) { - i2 = features[i2].renamed; + while (features[i2]->renamed >= 0) { + i2 = features[i2]->renamed; } - for (size_t j = 0; j < features[i].arc_polygon.size() && merged < merge_count; j++) { - if (features[i].arc_polygon[j] == order[o].edge) { + for (size_t j = 0; j < features[i]->arc_polygon.size() && merged < merge_count; j++) { + if (features[i]->arc_polygon[j] == order[o].edge) { { // XXX snap links - if (features[order[o].p2].arc_polygon.size() > 0) { + if (features[order[o].p2]->arc_polygon.size() > 0) { // This has to merge the ring that contains the anti-arc to this arc // into the current ring, and then add whatever other rings were in // that feature on to the end. @@ -427,11 +427,11 @@ bool find_common_edges(std::vector &features, int z, int line_de // the rings in order, but Wagyu should sort that out later std::vector additions; - std::vector &here = features[i].arc_polygon; - std::vector &other = features[i2].arc_polygon; + std::vector &here = features[i]->arc_polygon; + std::vector &other = features[i2]->arc_polygon; #if 0 - printf("seeking %zd\n", features[i].arc_polygon[j]); + printf("seeking %zd\n", features[i]->arc_polygon[j]); printf("before: "); for (size_t k = 0; k < here.size(); k++) { printf("%zd ", here[k]); @@ -464,7 +464,7 @@ bool find_common_edges(std::vector &features, int z, int line_de size_t m; for (m = k; m <= l; m++) { - if (other[m] == -features[i].arc_polygon[j]) { + if (other[m] == -features[i]->arc_polygon[j]) { break; } } @@ -494,12 +494,12 @@ bool find_common_edges(std::vector &features, int z, int line_de k = l; } - features[i2].arc_polygon.clear(); - features[i2].renamed = i; + features[i2]->arc_polygon.clear(); + features[i2]->renamed = i; merged++; for (size_t k = 0; k < additions.size(); k++) { - features[i].arc_polygon.push_back(additions[k]); + features[i]->arc_polygon.push_back(additions[k]); } #if 0 @@ -528,37 +528,37 @@ bool find_common_edges(std::vector &features, int z, int line_de // Turn the arc representations of the polygons back into standard polygon geometries for (size_t i = 0; i < features.size(); i++) { - if (features[i].t == VT_POLYGON) { - features[i].geometry.clear(); + if (features[i]->t == VT_POLYGON) { + features[i]->geometry.clear(); bool at_start = true; draw first(-1, 0, 0); - for (size_t j = 0; j < features[i].arc_polygon.size(); j++) { - ssize_t p = features[i].arc_polygon[j]; + for (size_t j = 0; j < features[i]->arc_polygon.size(); j++) { + ssize_t p = features[i]->arc_polygon[j]; if (p == 0) { if (first.op >= 0) { - features[i].geometry.push_back(first); + features[i]->geometry.push_back(first); first = draw(-1, 0, 0); } at_start = true; } else if (p > 0) { for (size_t k = 0; k + 1 < simplified_arcs[p].size(); k++) { if (at_start) { - features[i].geometry.push_back(draw(VT_MOVETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y)); + features[i]->geometry.push_back(draw(VT_MOVETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y)); first = draw(VT_LINETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y); } else { - features[i].geometry.push_back(draw(VT_LINETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y)); + features[i]->geometry.push_back(draw(VT_LINETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y)); } at_start = 0; } } else { /* p < 0 */ for (ssize_t k = simplified_arcs[-p].size() - 1; k > 0; k--) { if (at_start) { - features[i].geometry.push_back(draw(VT_MOVETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y)); + features[i]->geometry.push_back(draw(VT_MOVETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y)); first = draw(VT_LINETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y); } else { - features[i].geometry.push_back(draw(VT_LINETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y)); + features[i]->geometry.push_back(draw(VT_LINETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y)); } at_start = 0; } diff --git a/shared_borders.hpp b/shared_borders.hpp index 58771cdc..af3e7d0d 100644 --- a/shared_borders.hpp +++ b/shared_borders.hpp @@ -1 +1 @@ -bool find_common_edges(std::vector &features, int z, int line_detail, double simplification, int maxzoom, double merge_fraction); +bool find_common_edges(std::vector> &features, int z, int line_detail, double simplification, int maxzoom, double merge_fraction); diff --git a/tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json b/tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json index 9328c90c..a002d18f 100644 --- a/tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json +++ b/tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json @@ -5,7 +5,7 @@ "description": "tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json.check.mbtiles", "format": "pbf", "generator_options": "./tippecanoe -q -a@ -f -o tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json.check.mbtiles -z5 -Ethesum:sum -Etheproduct:product -Ethemax:max -Ethemin:min -Ethemean:mean -Etheconcat:concat -Ethecomma:comma tests/accumulate/in.json", -"json": "{\"vector_layers\":[{\"id\":\"in\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":5,\"fields\":{\"thecomma\":\"Mixed\",\"theconcat\":\"Mixed\",\"themax\":\"Number\",\"themean\":\"Number\",\"themin\":\"Number\",\"theproduct\":\"Number\",\"thesum\":\"Number\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"in\",\"count\":100,\"geometry\":\"Point\",\"attributeCount\":7,\"attributes\":[{\"attribute\":\"thecomma\",\"count\":152,\"type\":\"mixed\",\"values\":[1,\"1,8,44,6\",10,100,11,12,13,14,15,16,\"16,82\",17,\"17,62\",18,\"18,39\",\"18,39,15\",19,2,\"2,16,82\",20,21,22,23,24,\"24,83\",25,26,\"26,89,25\",\"26,89,25,55,91,33,17,62,69,81,7,93,31\",27,28,29,3,\"3,14\",\"3,14,78,28\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,73,33\",30,31,32,33,34,35,\"35,79\",\"35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",36,37,38,\"38,49\",\"38,49,43,54,71,9,32\",39,4,40,41,42,43,\"43,54\",44,45,46,47,48,49,\"49,1,31,70,64,8,44,6\",\"49,72,86,46,76,48,18,39,15,80,85,24,83,10\",5,\"5,12\",50,51,52,53,54,55,\"55,91\",\"55,91,33\",56,57,\"57,37\",\"57,37,97\",58,59,6,60,61,\"61,36\",62,63,64,65,\"65,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,96\",66,\"66,22\",\"66,22,63,95,73,86,19\",67,\"67,38,11,2,16,82\",\"67,65\",\"67,74,65,5,12,94,84,51,38\",68,69,\"69,81\"],\"min\":1,\"max\":100},{\"attribute\":\"theconcat\",\"count\":152,\"type\":\"mixed\",\"values\":[1,10,100,11,12,13,14,15,16,\"1682\",17,\"1762\",18,\"1839\",\"183915\",\"18446\",19,2,20,21,\"21682\",22,23,24,\"2483\",25,26,\"268925\",\"2689255591331762698179331\",27,28,29,3,30,31,\"314\",\"3147828\",\"31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658\",\"3147828677465512948451573797539299902113387333\",32,33,34,35,\"3579\",\"357913170648446526136879658\",36,37,38,\"3849\",\"3849435471932\",39,4,40,41,42,43,\"4354\",44,45,46,47,48,49,\"4913170648446\",\"4972864676481839158085248310\",5,50,51,\"512\",52,53,54,55,\"5591\",\"559133\",56,57,\"5737\",\"573797\",58,59,6,60,61,\"6136\",62,63,64,65,\"6538494354719327523722959421006622639573861996\",66,\"6622\",\"66226395738619\",67,\"67381121682\",\"6765\",\"67746551294845138\",68,69,\"6981\"],\"min\":1,\"max\":100},{\"attribute\":\"themax\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"themean\",\"count\":134,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,14.75,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,28.5,29,3,30,30.75,31,32,33,33.333333333333339,34,34.125,35,36,37,37.333333333333339,38,39,39.5,4,40,41,42,42.285714285714288,43,43.5,44,45,46,46.666666666666667,47,47.333333333333339,48,48.5,48.53333333333333,49,5,50,50.5,50.773809523809529,51,52,52.214285714285718,52.23076923076923,52.333333333333339,53,53.25,53.5,54,54.44444444444444,55,55.69565217391305,55.75,56,57,58,59,59.333333333333339,59.666666666666667,6,60,60.4,60.57142857142857,61,62,62.5,63,63.666666666666667,64,64.66666666666667,65,66,67,68,69,69.33333333333333,7,70],\"min\":1,\"max\":100},{\"attribute\":\"themin\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"theproduct\",\"count\":150,\"type\":\"number\",\"values\":[1,1.1358965693283625e+21,1.5864380887725077e+38,1.6098219495357537e+132,10,100,1036586822040,10530,1054,11,119282,12,12157085491200,13,1312,1320551424,14,14372413440,1452,15,16,16336199880,165165,1693200,17,1725,18,1862,19,1992,2,20,20448,204573,2088,21,2109,2112,2196,22,23,2322,24,2432430,25,26,2624,27,2765,28,288600,29,295891195017600,3,30,300656,31,32,33,34,35,35840,36,3638439,37,38,39,39123,3982783094784,4,4.486244263022014e+22,40,402696,41,42,43,4355,44,45,46,47,48,49,5,5.103920456870841e+38,50,5005,51,517406400,52,53,54,55,5589,56,57,57850,58,59,6],\"min\":1,\"max\":1.6098219495357537e+132},{\"attribute\":\"thesum\",\"count\":136,\"type\":\"number\",\"values\":[1,10,100,101,107,11,112,114,12,123,1278,1281,13,132,14,140,142,146,15,150,156,157,16,17,178,179,18,19,191,2,20,202,208,21,216,22,223,229,23,24,25,250,26,27,273,28,29,296,3,30,302,31,32,33,34,35,352,36,37,38,388,39,4,40,408,41,42,424,4265,43,44,45,46,462,47,48,49,490,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,679,68],\"min\":1,\"max\":4265}]}]}}", +"json": "{\"vector_layers\":[{\"id\":\"in\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":5,\"fields\":{\"thecomma\":\"Mixed\",\"theconcat\":\"Mixed\",\"themax\":\"Number\",\"themean\":\"Number\",\"themin\":\"Number\",\"theproduct\":\"Number\",\"thesum\":\"Number\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"in\",\"count\":100,\"geometry\":\"Point\",\"attributeCount\":7,\"attributes\":[{\"attribute\":\"thecomma\",\"count\":152,\"type\":\"mixed\",\"values\":[1,\"1,8,44,6\",10,100,11,12,13,14,15,16,\"16,82\",17,\"17,62\",18,\"18,39\",\"18,39,15\",19,2,\"2,16,82\",20,21,22,23,24,\"24,83\",25,26,\"26,89,25\",\"26,89,25,55,91,33,17,62,69,81,7,93,31\",27,28,29,3,\"3,14\",\"3,14,78,28\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,73,33\",30,31,32,33,34,35,\"35,79\",\"35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",36,37,38,\"38,49\",\"38,49,43,54,71,9,32\",39,4,40,41,42,43,\"43,54\",44,45,46,47,48,49,\"49,1,31,70,64,8,44,6\",\"49,72,86,46,76,48,18,39,15,80,85,24,83,10\",5,\"5,12\",50,51,52,53,54,55,\"55,91\",\"55,91,33\",56,57,\"57,37\",\"57,37,97\",58,59,6,60,61,\"61,36\",62,63,64,65,\"65,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,96\",66,\"66,22\",\"66,22,63,95,73,86,19\",67,\"67,38,11,2,16,82\",\"67,65\",\"67,74,65,5,12,94,84,51,38\",68,69,\"69,81\"],\"min\":1,\"max\":100},{\"attribute\":\"theconcat\",\"count\":152,\"type\":\"mixed\",\"values\":[1,10,100,11,12,13,14,15,16,\"1682\",17,\"1762\",18,\"1839\",\"183915\",\"18446\",19,2,20,21,\"21682\",22,23,24,\"2483\",25,26,\"268925\",\"2689255591331762698179331\",27,28,29,3,30,31,\"314\",\"3147828\",\"31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658\",\"3147828677465512948451573797539299902113387333\",32,33,34,35,\"3579\",\"357913170648446526136879658\",36,37,38,\"3849\",\"3849435471932\",39,4,40,41,42,43,\"4354\",44,45,46,47,48,49,\"4913170648446\",\"4972864676481839158085248310\",5,50,51,\"512\",52,53,54,55,\"5591\",\"559133\",56,57,\"5737\",\"573797\",58,59,6,60,61,\"6136\",62,63,64,65,\"6538494354719327523722959421006622639573861996\",66,\"6622\",\"66226395738619\",67,\"67381121682\",\"6765\",\"67746551294845138\",68,69,\"6981\"],\"min\":1,\"max\":100},{\"attribute\":\"themax\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"themean\",\"count\":134,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,14.75,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,28.5,29,3,30,30.75,31,32,33,33.333333333333339,34,34.125,35,36,37,37.333333333333339,38,39,39.5,4,40,41,42,42.285714285714288,43,43.5,44,45,46,46.666666666666667,47,47.333333333333339,48,48.5,48.53333333333333,49,5,50,50.5,50.77380952380951,51,52,52.214285714285718,52.23076923076923,52.333333333333339,53,53.25,53.5,54,54.44444444444444,55,55.69565217391305,55.75,56,57,58,59,59.333333333333339,59.666666666666667,6,60,60.4,60.57142857142857,61,62,62.5,63,63.666666666666667,64,64.66666666666667,65,66,67,68,69,69.33333333333333,7,70],\"min\":1,\"max\":100},{\"attribute\":\"themin\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"theproduct\",\"count\":150,\"type\":\"number\",\"values\":[1,1.1358965693283625e+21,1.5864380887725077e+38,1.6098219495357537e+132,10,100,1036586822040,10530,1054,11,119282,12,12157085491200,13,1312,1320551424,14,14372413440,1452,15,16,16336199880,165165,1693200,17,1725,18,1862,19,1992,2,20,20448,204573,2088,21,2109,2112,2196,22,23,2322,24,2432430,25,26,2624,27,2765,28,288600,29,295891195017600,3,30,300656,31,32,33,34,35,35840,36,3638439,37,38,39,39123,3982783094784,4,4.486244263022014e+22,40,402696,41,42,43,4355,44,45,46,47,48,49,5,5.103920456870841e+38,50,5005,51,517406400,52,53,54,55,5589,56,57,57850,58,59,6],\"min\":1,\"max\":1.6098219495357537e+132},{\"attribute\":\"thesum\",\"count\":136,\"type\":\"number\",\"values\":[1,10,100,101,107,11,112,114,12,123,1278,1281,13,132,14,140,142,146,15,150,156,157,16,17,178,179,18,19,191,2,20,202,208,21,216,22,223,229,23,24,25,250,26,27,273,28,29,296,3,30,302,31,32,33,34,35,352,36,37,38,388,39,4,40,408,41,42,424,4265,43,44,45,46,462,47,48,49,490,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,679,68],\"min\":1,\"max\":4265}]}]}}", "maxzoom": "5", "minzoom": "0", "name": "tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json.check.mbtiles", @@ -15,7 +15,7 @@ }, "features": [ { "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "thesum": 4265, "themax": 100, "themin": 1, "theproduct": 1.6098219495357537e+132, "themean": 50.773809523809529, "theconcat": "31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658", "thecomma": "3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58" }, "geometry": { "type": "Point", "coordinates": [ -147.392578, 84.778533 ] } } +{ "type": "Feature", "properties": { "thesum": 4265, "themax": 100, "themin": 1, "theproduct": 1.6098219495357537e+132, "themean": 50.77380952380951, "theconcat": "31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658", "thecomma": "3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58" }, "geometry": { "type": "Point", "coordinates": [ -147.392578, 84.778533 ] } } ] } ] } , diff --git a/tile.cpp b/tile.cpp index 108da534..0703ff97 100644 --- a/tile.cpp +++ b/tile.cpp @@ -92,13 +92,17 @@ static bool draws_something(drawvec const &geom) { // comparator for --preserve-input-order, to reorder features back to their original input sequence static struct preservecmp { - bool operator()(const std::vector &a, const std::vector &b) { + bool operator()(const std::vector> &a, const std::vector> &b) { return operator()(a[0], b[0]); } bool operator()(const serial_feature &a, const serial_feature &b) { return a.seq < b.seq; } + + bool operator()(const std::shared_ptr &a, const std::shared_ptr &b) { + return a->seq < b->seq; + } } preservecmp; static int metacmp(const serial_feature &one, const serial_feature &two); @@ -190,8 +194,8 @@ struct coalindexcmp_comparator { return cmp; } - bool operator()(const serial_feature &a, const serial_feature &o) const { - int cmp = coalindexcmp(&a, &o); + bool operator()(const std::shared_ptr &a, const std::shared_ptr &o) const { + int cmp = coalindexcmp(&*a, &*o); if (cmp < 0) { return true; } else { @@ -203,9 +207,9 @@ struct coalindexcmp_comparator { static unsigned long long calculate_drop_sequence(serial_feature const &sf); struct drop_sequence_cmp { - bool operator()(const serial_feature &a, const serial_feature &b) { - unsigned long long a_seq = calculate_drop_sequence(a); - unsigned long long b_seq = calculate_drop_sequence(b); + bool operator()(const std::shared_ptr &a, const std::shared_ptr &b) { + unsigned long long a_seq = calculate_drop_sequence(*a); + unsigned long long b_seq = calculate_drop_sequence(*b); // sorts backwards, to put the features that would be dropped last, first here if (a_seq > b_seq) { @@ -336,14 +340,14 @@ static mvt_value coerce_double(mvt_value v) { // compare features numerically according to that sort key until the keys are exhausted. // If there is a tie, the feature with the earlier index (centroid) comes first. struct ordercmp { - bool operator()(const std::vector &a, const std::vector &b) { + bool operator()(const std::vector> &a, const std::vector> &b) { return operator()(a[0], b[0]); } - bool operator()(const serial_feature &a, const serial_feature &b) { + bool operator()(const std::shared_ptr &a, const std::shared_ptr &b) { for (size_t i = 0; i < order_by.size(); i++) { - mvt_value v1 = coerce_double(find_attribute_value(&a, order_by[i].name)); - mvt_value v2 = coerce_double(find_attribute_value(&b, order_by[i].name)); + mvt_value v1 = coerce_double(find_attribute_value(&*a, order_by[i].name)); + mvt_value v2 = coerce_double(find_attribute_value(&*b, order_by[i].name)); if (order_by[i].descending) { if (v2 < v1) { @@ -360,7 +364,7 @@ struct ordercmp { } } - if (a.index < b.index) { + if (a->index < b->index) { return true; } @@ -370,12 +374,12 @@ struct ordercmp { // For --retain-points-multiplier: Go through a list of features and return a list of clusters of features, // creating a new cluster whenever the tippecanoe:retain_points_multiplier_first attribute is seen. -static std::vector> assemble_multiplier_clusters(std::vector const &features) { - std::vector> clusters; +static std::vector>> assemble_multiplier_clusters(std::vector> const &features) { + std::vector>> clusters; if (retain_points_multiplier == 1) { for (auto const &feature : features) { - std::vector cluster; + std::vector> cluster; cluster.push_back(std::move(feature)); clusters.push_back(std::move(cluster)); } @@ -383,8 +387,8 @@ static std::vector> assemble_multiplier_clusters(std for (auto const &feature : features) { bool is_cluster_start = false; - for (size_t i = 0; i < feature.full_keys.size(); i++) { - if (*feature.full_keys[i] == "tippecanoe:retain_points_multiplier_first") { + for (size_t i = 0; i < feature->full_keys.size(); i++) { + if (*feature->full_keys[i] == "tippecanoe:retain_points_multiplier_first") { is_cluster_start = true; break; } @@ -404,20 +408,20 @@ static std::vector> assemble_multiplier_clusters(std // For --retain-points-multiplier: Flatten a list of clusters of features back into a list of features, // moving the "tippecanoe:retain_points_multiplier_first" attribute onto the first feature of each cluster // if it is not already there. -static std::vector disassemble_multiplier_clusters(std::vector> &clusters) { - std::vector out; +static std::vector> disassemble_multiplier_clusters(std::vector>> &clusters) { + std::vector> out; for (auto &cluster : clusters) { // fix up the attributes so the first feature of the multiplier cluster // gets the marker attribute for (size_t i = 0; i < cluster.size(); i++) { - for (size_t j = 0; j < cluster[i].full_keys.size(); j++) { - if (*cluster[i].full_keys[j] == "tippecanoe:retain_points_multiplier_first") { - cluster[0].full_keys.push_back(std::move(cluster[i].full_keys[j])); - cluster[0].full_values.push_back(std::move(cluster[i].full_values[j])); + for (size_t j = 0; j < cluster[i]->full_keys.size(); j++) { + if (*cluster[i]->full_keys[j] == "tippecanoe:retain_points_multiplier_first") { + cluster[0]->full_keys.push_back(std::move(cluster[i]->full_keys[j])); + cluster[0]->full_values.push_back(std::move(cluster[i]->full_values[j])); - cluster[i].full_keys.erase(cluster[i].full_keys.begin() + j); - cluster[i].full_values.erase(cluster[i].full_values.begin() + j); + cluster[i]->full_keys.erase(cluster[i]->full_keys.begin() + j); + cluster[i]->full_values.erase(cluster[i]->full_values.begin() + j); i = cluster.size(); // break outer break; @@ -534,7 +538,7 @@ static void rewrite(serial_feature const &osf, int z, int nextzoom, int maxzoom, // This is the parameter block passed to each simplification worker thread struct simplification_worker_arg { - std::vector *features = NULL; + std::vector> *features = NULL; int task = 0; int tasks = 0; bool trying_to_stop_early = false; @@ -668,19 +672,19 @@ static double simplify_feature(serial_feature *p, drawvec const &shared_nodes, n // simplify and clean the geometry of batches of features. static void *simplification_worker(void *v) { simplification_worker_arg *a = (simplification_worker_arg *) v; - std::vector *features = a->features; + std::vector> *features = a->features; for (size_t i = a->task; i < (*features).size(); i += a->tasks) { double area = 0; if (!a->trying_to_stop_early) { - area = simplify_feature(&((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos, *(a->shared_nodes_bloom)); + area = simplify_feature(&*((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos, *(a->shared_nodes_bloom)); } - signed char t = (*features)[i].t; - int z = (*features)[i].z; - int out_detail = (*features)[i].extra_detail; + signed char t = (*features)[i]->t; + int z = (*features)[i]->z; + int out_detail = (*features)[i]->extra_detail; - drawvec geom = (*features)[i].geometry; + drawvec geom = (*features)[i]->geometry; to_tile_scale(geom, z, out_detail); if (t == VT_POLYGON) { @@ -709,15 +713,15 @@ static void *simplification_worker(void *v) { } if (t == VT_POLYGON && additional[A_GENERATE_POLYGON_LABEL_POINTS]) { - t = (*features)[i].t = VT_POINT; - geom = checkerboard_anchors(from_tile_scale(geom, z, out_detail), (*features)[i].tx, (*features)[i].ty, z, (*features)[i].label_point); + t = (*features)[i]->t = VT_POINT; + geom = checkerboard_anchors(from_tile_scale(geom, z, out_detail), (*features)[i]->tx, (*features)[i]->ty, z, (*features)[i]->label_point); to_tile_scale(geom, z, out_detail); } - if ((*features)[i].index == 0) { - (*features)[i].index = i; + if ((*features)[i]->index == 0) { + (*features)[i]->index = i; } - (*features)[i].geometry = std::move(geom); + (*features)[i]->geometry = std::move(geom); } return NULL; @@ -1500,7 +1504,7 @@ void preserve_attributes(std::unordered_map const *at sv.s = sf.stringpool + sf.values[i] + 1; promote_attribute(key, p, key_pool); - preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state, key_pool); + preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, key_pool); } else if (type == mvt_double && accumulate_numeric.size() > 0 && !starts_with(key, accumulate_numeric_colon)) { for (auto const &operation : numeric_operations) { serial_val sv; @@ -1509,7 +1513,7 @@ void preserve_attributes(std::unordered_map const *at std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key; promote_attribute_prefix(key, prefixed_key, p, key_pool); - preserve_attribute(operation.second, prefixed_key, sv, p.full_keys, p.full_values, p.attribute_accum_state, key_pool); + preserve_attribute(operation.second, prefixed_key, sv, p.full_keys, p.full_values, key_pool); } } } @@ -1522,12 +1526,12 @@ void preserve_attributes(std::unordered_map const *at const serial_val &sv = sf.full_values[i]; promote_attribute(key, p, key_pool); // promotes it in the target feature - preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state, key_pool); + preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, key_pool); } else if (type == mvt_double && accumulate_numeric.size() > 0 && !starts_with(key, accumulate_numeric_colon)) { for (auto const &operation : numeric_operations) { std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key; promote_attribute_prefix(key, prefixed_key, p, key_pool); - preserve_attribute(operation.second, prefixed_key, sf.full_values[i], p.full_keys, p.full_values, p.attribute_accum_state, key_pool); + preserve_attribute(operation.second, prefixed_key, sf.full_values[i], p.full_keys, p.full_values, key_pool); } } } @@ -1538,13 +1542,13 @@ void preserve_attributes(std::unordered_map const *at // ordinarily returns the most recently-added feature from the same layer as the feature // that is being dropped. // -bool find_feature_to_accumulate_onto(std::vector &features, serial_feature &sf, ssize_t &out, std::vector> *layer_unmaps, long long maxextent) { +bool find_feature_to_accumulate_onto(std::vector> &features, serial_feature &sf, ssize_t &out, std::vector> *layer_unmaps, long long maxextent) { for (size_t i = features.size(); i > 0; i--) { - if (features[i - 1].t == sf.t) { - std::string &layername1 = (*layer_unmaps)[features[i - 1].segment][features[i - 1].layer]; + if (features[i - 1]->t == sf.t) { + std::string &layername1 = (*layer_unmaps)[features[i - 1]->segment][features[i - 1]->layer]; std::string &layername2 = (*layer_unmaps)[sf.segment][sf.layer]; - if (layername1 == layername2 && features[i - 1].extent <= maxextent) { + if (layername1 == layername2 && features[i - 1]->extent <= maxextent) { out = i - 1; return true; } @@ -1617,8 +1621,8 @@ return; // This is the structure that the features from each layer are accumulated into struct layer_features { - std::vector features; // The features of this layer, so far - size_t multiplier_cluster_size = 0; // The feature count of the current multiplier cluster + std::vector> features; // The features of this layer, so far + size_t multiplier_cluster_size = 0; // The feature count of the current multiplier cluster }; bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features &layer, serial_feature &sf, std::vector> *layer_unmaps, strategy &strategy, bool &drop_rest, std::unordered_map const *attribute_accum, key_pool &key_pool) { @@ -1632,7 +1636,7 @@ bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features sf.dropped = layer.multiplier_cluster_size + 1; return false; // converted rather than dropped } else { - preserve_attributes(attribute_accum, sf, layer.features[which_serial_feature], key_pool); + preserve_attributes(attribute_accum, sf, *layer.features[which_serial_feature], key_pool); drop_rest = true; return true; // dropped } @@ -1872,7 +1876,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch layers.emplace(layername, layer_features()); } struct layer_features &layer = layers.find(layername)->second; - std::vector &features = layer.features; + std::vector> &features = layer.features; if (sf.t == VT_POINT) { if (extent_previndex >= sf.index) { @@ -1921,7 +1925,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (sf.dropped == FEATURE_DROPPED || drop_rest) { if (find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); + preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool); strategy.dropped_by_rate++; can_stop_early = false; continue; @@ -1934,7 +1938,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (sf.dropped == FEATURE_KEPT) { if (gamma > 0) { if (manage_gap(sf.index, &previndex, scale, gamma, &gap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); + preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool); strategy.dropped_by_gamma++; drop_rest = true; can_stop_early = false; @@ -1949,20 +1953,20 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch // rather than wanting each feature to have a consistent // idea of density between zooms. if ((sf.index < merge_previndex || sf.index - merge_previndex < cluster_mingap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { - features[which_serial_feature].clustered++; + features[which_serial_feature]->clustered++; - if (features[which_serial_feature].t == VT_POINT && - features[which_serial_feature].geometry.size() == 1 && + if (features[which_serial_feature]->t == VT_POINT && + features[which_serial_feature]->geometry.size() == 1 && sf.geometry.size() == 1) { - double x = (double) features[which_serial_feature].geometry[0].x * features[which_serial_feature].clustered; - double y = (double) features[which_serial_feature].geometry[0].y * features[which_serial_feature].clustered; + double x = (double) features[which_serial_feature]->geometry[0].x * features[which_serial_feature]->clustered; + double y = (double) features[which_serial_feature]->geometry[0].y * features[which_serial_feature]->clustered; x += sf.geometry[0].x; y += sf.geometry[0].y; - features[which_serial_feature].geometry[0].x = x / (features[which_serial_feature].clustered + 1); - features[which_serial_feature].geometry[0].y = y / (features[which_serial_feature].clustered + 1); + features[which_serial_feature]->geometry[0].x = x / (features[which_serial_feature]->clustered + 1); + features[which_serial_feature]->geometry[0].y = y / (features[which_serial_feature]->clustered + 1); } - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); + preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; @@ -1981,20 +1985,20 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch // it averages the point locations add_sample_to(gaps, sf.gap, gaps_increment, seq); if (sf.gap < mingap && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { - features[which_serial_feature].clustered++; + features[which_serial_feature]->clustered++; - if (features[which_serial_feature].t == VT_POINT && - features[which_serial_feature].geometry.size() == 1 && + if (features[which_serial_feature]->t == VT_POINT && + features[which_serial_feature]->geometry.size() == 1 && sf.geometry.size() == 1) { - double x = (double) features[which_serial_feature].geometry[0].x * features[which_serial_feature].clustered; - double y = (double) features[which_serial_feature].geometry[0].y * features[which_serial_feature].clustered; + double x = (double) features[which_serial_feature]->geometry[0].x * features[which_serial_feature]->clustered; + double y = (double) features[which_serial_feature]->geometry[0].y * features[which_serial_feature]->clustered; x += sf.geometry[0].x; y += sf.geometry[0].y; - features[which_serial_feature].geometry[0].x = x / (features[which_serial_feature].clustered + 1); - features[which_serial_feature].geometry[0].y = y / (features[which_serial_feature].clustered + 1); + features[which_serial_feature]->geometry[0].x = x / (features[which_serial_feature]->clustered + 1); + features[which_serial_feature]->geometry[0].y = y / (features[which_serial_feature]->clustered + 1); } - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); + preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; continue; @@ -2002,10 +2006,10 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } else if (additional[A_COALESCE_DENSEST_AS_NEEDED]) { add_sample_to(gaps, sf.gap, gaps_increment, seq); if (sf.gap < mingap && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { - coalesce_geometry(features[which_serial_feature], sf); - features[which_serial_feature].coalesced = true; + coalesce_geometry(*features[which_serial_feature], sf); + features[which_serial_feature]->coalesced = true; coalesced_area += sf.extent; - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); + preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; @@ -2024,10 +2028,10 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } else if (additional[A_COALESCE_SMALLEST_AS_NEEDED]) { add_sample_to(extents, sf.extent, extents_increment, seq); if (minextent != 0 && sf.extent + coalesced_area <= minextent && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, minextent)) { - coalesce_geometry(features[which_serial_feature], sf); - features[which_serial_feature].coalesced = true; + coalesce_geometry(*features[which_serial_feature], sf); + features[which_serial_feature]->coalesced = true; coalesced_area += sf.extent; - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); + preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; @@ -2044,9 +2048,9 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } else if (additional[A_COALESCE_FRACTION_AS_NEEDED]) { add_sample_to(drop_sequences, drop_sequence, drop_sequences_increment, seq); if (mindrop_sequence != 0 && drop_sequence <= mindrop_sequence && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { - coalesce_geometry(features[which_serial_feature], sf); - features[which_serial_feature].coalesced = true; - preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); + coalesce_geometry(*features[which_serial_feature], sf); + features[which_serial_feature]->coalesced = true; + preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool); strategy.coalesced_as_needed++; drop_rest = true; can_stop_early = false; @@ -2188,23 +2192,23 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch tile_detail = sf.extra_detail; } - features.push_back(std::move(sf)); + features.push_back(std::make_shared(sf)); - unsimplified_geometry_size += features.back().geometry.size() * sizeof(draw); + unsimplified_geometry_size += features.back()->geometry.size() * sizeof(draw); if (unsimplified_geometry_size > 10 * 1024 * 1024 && !additional[A_DETECT_SHARED_BORDERS]) { // we should be safe to simplify here with P_SIMPLIFY_SHARED_NODES, since they will // have been assembled globally, although that also means that simplification // may not be very effective for reducing memory usage. for (; simplified_geometry_through < features.size(); simplified_geometry_through++) { - simplify_feature(&features[simplified_geometry_through], shared_nodes, shared_nodes_map, nodepos, shared_nodes_bloom); + simplify_feature(&*features[simplified_geometry_through], shared_nodes, shared_nodes_map, nodepos, shared_nodes_bloom); - if (features[simplified_geometry_through].t == VT_POLYGON) { - drawvec to_clean = features[simplified_geometry_through].geometry; + if (features[simplified_geometry_through]->t == VT_POLYGON) { + drawvec to_clean = features[simplified_geometry_through]->geometry; // don't scale up because this is still world coordinates to_clean = clean_or_clip_poly(to_clean, 0, 0, false, false); - features[simplified_geometry_through].geometry = std::move(to_clean); + features[simplified_geometry_through]->geometry = std::move(to_clean); } } @@ -2293,7 +2297,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch for (auto &kv : layers) { std::string const &layername = kv.first; - std::vector &features = kv.second.features; + std::vector> &features = kv.second.features; if (retain_points_multiplier > 1) { add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "tippecanoe:retain_points_multiplier_first", serial_val(mvt_bool, "true")); @@ -2302,7 +2306,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch std::vector> feature_sequences; for (size_t i = 0; i < features.size(); i++) { - feature_sequences.emplace_back(features[i].seq, i); + feature_sequences.emplace_back(features[i]->seq, i); } // tag each feature with its sequence number within the layer @@ -2316,15 +2320,15 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch size_t j = feature_sequences[i].second; serial_val sv(mvt_double, std::to_string(i)); - features[j].full_keys.push_back(key_pool.pool("tippecanoe:retain_points_multiplier_sequence")); - features[j].full_values.push_back(sv); + features[j]->full_keys.push_back(key_pool.pool("tippecanoe:retain_points_multiplier_sequence")); + features[j]->full_values.push_back(sv); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *features[j].full_keys.back(), sv); + add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *features[j]->full_keys.back(), sv); } } for (size_t i = 0; i < features.size(); i++) { - serial_feature &p = features[i]; + serial_feature &p = *features[i]; if (p.clustered > 0) { serial_val sv, sv2, sv3, sv4; @@ -2370,6 +2374,11 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (p.need_tilestats.size() > 0) { for (size_t j = 0; j < p.full_keys.size(); j++) { if (p.need_tilestats.count(*p.full_keys[j]) > 0) { + // remove accumulation state + size_t found = p.full_values[j].s.find('\0'); + if (found != std::string::npos) { + p.full_values[j].s = p.full_values[j].s.substr(0, found); + } add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]); } } @@ -2421,18 +2430,18 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } for (size_t i = 0; i < features.size(); i++) { - signed char t = features[i].t; + signed char t = features[i]->t; { - if (t == VT_POINT || draws_something(features[i].geometry)) { + if (t == VT_POINT || draws_something(features[i]->geometry)) { // printf("segment %d layer %lld is %s\n", features[i].segment, features[i].layer, (*layer_unmaps)[features[i].segment][features[i].layer].c_str()); - features[i].coalesced = false; + features[i]->coalesced = false; } } } - std::vector &layer_features = features; + std::vector> &layer_features = features; if (additional[A_REORDER]) { std::stable_sort(layer_features.begin(), layer_features.end(), coalindexcmp_comparator()); @@ -2449,11 +2458,11 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch for (size_t x = 1; x < layer_features.size(); x++) { size_t y = out - 1; - if (out > 0 && coalcmp(&layer_features[x], &layer_features[y]) == 0) { - for (size_t g = 0; g < layer_features[x].geometry.size(); g++) { - layer_features[y].geometry.push_back(std::move(layer_features[x].geometry[g])); + if (out > 0 && coalcmp(&*layer_features[x], &*layer_features[y]) == 0) { + for (size_t g = 0; g < layer_features[x]->geometry.size(); g++) { + layer_features[y]->geometry.push_back(std::move(layer_features[x]->geometry[g])); } - layer_features[y].coalesced = true; + layer_features[y]->coalesced = true; } else { layer_features[out++] = layer_features[x]; } @@ -2471,25 +2480,25 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch size_t out = 0; for (size_t x = 0; x < layer_features.size(); x++) { - if (layer_features[x].coalesced && layer_features[x].t == VT_LINE) { - layer_features[x].geometry = remove_noop(layer_features[x].geometry, layer_features[x].t, 0); + if (layer_features[x]->coalesced && layer_features[x]->t == VT_LINE) { + layer_features[x]->geometry = remove_noop(layer_features[x]->geometry, layer_features[x]->t, 0); if (!(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]))) { // XXX revisit: why does this not take zoom into account? - layer_features[x].geometry = simplify_lines(layer_features[x].geometry, 32, 0, 0, 0, - !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, layer_features[x].t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, ""); + layer_features[x]->geometry = simplify_lines(layer_features[x]->geometry, 32, 0, 0, 0, + !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, layer_features[x]->t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, ""); } } - if (layer_features[x].t == VT_POLYGON) { - if (layer_features[x].coalesced) { + if (layer_features[x]->t == VT_POLYGON) { + if (layer_features[x]->coalesced) { // we can try scaling up because this is tile coordinates - layer_features[x].geometry = clean_or_clip_poly(layer_features[x].geometry, 0, 0, false, true); + layer_features[x]->geometry = clean_or_clip_poly(layer_features[x]->geometry, 0, 0, false, true); } - layer_features[x].geometry = close_poly(layer_features[x].geometry); + layer_features[x]->geometry = close_poly(layer_features[x]->geometry); } - if (layer_features[x].geometry.size() > 0) { + if (layer_features[x]->geometry.size() > 0) { layer_features[out++] = layer_features[x]; } } @@ -2530,7 +2539,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch size_t feature_count = 0; for (auto layer_iterator = layers.begin(); layer_iterator != layers.end(); ++layer_iterator) { - std::vector &layer_features = layer_iterator->second.features; + std::vector> &layer_features = layer_iterator->second.features; feature_count += layer_features.size(); mvt_layer layer; @@ -2541,34 +2550,34 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch for (size_t x = 0; x < layer_features.size(); x++) { mvt_feature feature; - if (layer_features[x].t == VT_LINE || layer_features[x].t == VT_POLYGON) { - layer_features[x].geometry = remove_noop(layer_features[x].geometry, layer_features[x].t, 0); + if (layer_features[x]->t == VT_LINE || layer_features[x]->t == VT_POLYGON) { + layer_features[x]->geometry = remove_noop(layer_features[x]->geometry, layer_features[x]->t, 0); } - if (layer_features[x].geometry.size() == 0) { - layer_features[x] = serial_feature(); + if (layer_features[x]->geometry.size() == 0) { + layer_features[x] = std::make_shared(); continue; } - feature.type = layer_features[x].t; - feature.geometry = to_feature(layer_features[x].geometry); - count += layer_features[x].geometry.size(); - layer_features[x].geometry.clear(); + feature.type = layer_features[x]->t; + feature.geometry = to_feature(layer_features[x]->geometry); + count += layer_features[x]->geometry.size(); + layer_features[x]->geometry.clear(); - feature.id = layer_features[x].id; - feature.has_id = layer_features[x].has_id; + feature.id = layer_features[x]->id; + feature.has_id = layer_features[x]->has_id; - decode_meta(layer_features[x], layer, feature); - for (size_t a = 0; a < layer_features[x].full_keys.size(); a++) { - serial_val sv = layer_features[x].full_values[a]; + decode_meta(*layer_features[x], layer, feature); + for (size_t a = 0; a < layer_features[x]->full_keys.size(); a++) { + serial_val sv = layer_features[x]->full_values[a]; mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); - layer.tag(feature, *layer_features[x].full_keys[a], v); + layer.tag(feature, *layer_features[x]->full_keys[a], v); } if (additional[A_CALCULATE_FEATURE_DENSITY]) { int glow = 255; - if (layer_features[x].spacing > 0) { - glow = (1 / layer_features[x].spacing); + if (layer_features[x]->spacing > 0) { + glow = (1 / layer_features[x]->spacing); if (glow > 255) { glow = 255; } @@ -2587,7 +2596,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } layer.features.push_back(std::move(feature)); - layer_features[x] = serial_feature(); + layer_features[x] = std::make_shared(); } if (layer.features.size() > 0) { diff --git a/version.hpp b/version.hpp index 71bf5f69..4b0af2fb 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v2.74.0" +#define VERSION "v2.75.0" #endif