diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Reduce attribute accumulation memory consumption (#318)

Browse Source 
* Add a flag to use an H3 index for the feature index

* Give mvt_value and serial_val a double-with-count concept

* Switch mean over to internal accumulation state

* Get rid of the attribute accumulation map

* Change vectors of features to vectors of pointers to features

* Fix --coalesce

* Revert "Add a flag to use an H3 index for the feature index"

This reverts commit b9b48f42c9.

* Update version and changelog
This commit is contained in:
Erica Fischer
2025-01-30 21:58:35 -08:00
committed by GitHub
parent 390c362452
commit 583fc3744a
11 changed files with 243 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+4
View File
@@ -1,3 +1,7 @@
# 2.75.0
* Reduce memory consumption in attribute accumulation and feature sorting
# 2.74.0 # 2.74.0
* Add the option to join attributes from a sqlite database in tile-join * Add the option to join attributes from a sqlite database in tile-join
attribute.cpp
+17 -38
View File
@@ -89,7 +89,7 @@ void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribut
} }
template <class T> template <class T>
static void preserve_attribute1(attribute_op const &op, std::string const &key, T const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<T> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state, key_pool &key_pool) { static void preserve_attribute1(attribute_op const &op, std::string const &key, T const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<T> &full_values, key_pool &key_pool) {
for (size_t i = 0; i < full_keys.size(); i++) { for (size_t i = 0; i < full_keys.size(); i++) {
if (key == *full_keys[i]) { if (key == *full_keys[i]) {
switch (op) { switch (op) {
@@ -120,19 +120,13 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key,
} }
case op_mean: { case op_mean: {
auto state = attribute_accum_state.find(key); size_t count = full_values[i].get_count();
if (state == attribute_accum_state.end()) { if (count <= 1) {
accum_state s; full_values[i].set_double_count((full_values[i].to_double() + val.to_double()) / 2, 2);
s.sum = full_values[i].to_double() + val.to_double();
s.count = 2;
attribute_accum_state.insert(std::pair<std::string, accum_state>(key, s));
full_values[i] = (s.sum / s.count);
} else { } else {
state->second.sum += val.to_double(); double sum = full_values[i].to_double() * count + val.to_double();
state->second.count += 1; count++;
full_values[i].set_double_count(sum / count, count);
full_values[i] = (state->second.sum / state->second.count);
} }
return; return;
} }
@@ -146,16 +140,11 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key,
return; return;
case op_count: { case op_count: {
auto state = attribute_accum_state.find(key); size_t count = full_values[i].get_count();
if (state == attribute_accum_state.end()) { // not already present if (count <= 1) {
accum_state s; full_values[i].set_double_count(2, 2);
s.count = 2; } else {
attribute_accum_state.insert(std::pair<std::string, accum_state>(key, s)); full_values[i].set_double_count(count + 1, count + 1);
full_values[i] = (s.count);
} else { // already present, incrementing
state->second.count += 1;
full_values[i] = (state->second.count);
} }
return; return;
} }
@@ -174,17 +163,7 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key,
break; break;
case op_count: { case op_count: {
auto state = attribute_accum_state.find(key); v.set_double_count(1, 1);
if (state == attribute_accum_state.end()) { // not already present
accum_state s;
s.count = 1;
attribute_accum_state.insert(std::pair<std::string, accum_state>(key, s));
v = (s.count);
} else { // already present, incrementing
fprintf(stderr, "preserve_attribute: can't happen (count)\n");
exit(EXIT_IMPOSSIBLE);
}
break; break;
} }
@@ -197,10 +176,10 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key,
full_values.push_back(v); full_values.push_back(v);
} }
void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<mvt_value> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state, key_pool &key_pool) { void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<mvt_value> &full_values, key_pool &key_pool) {
preserve_attribute1(op, key, val, full_keys, full_values, attribute_accum_state, key_pool); preserve_attribute1(op, key, val, full_keys, full_values, key_pool);
} }
void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<serial_val> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state, key_pool &key_pool) { void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<serial_val> &full_values, key_pool &key_pool) {
preserve_attribute1(op, key, val, full_keys, full_values, attribute_accum_state, key_pool); preserve_attribute1(op, key, val, full_keys, full_values, key_pool);
} }
attribute.hpp
+2 -7
View File
@@ -19,19 +19,14 @@ enum attribute_op {
op_count, op_count,
}; };
struct accum_state {
double sum = 0;
double count = 0;
};
struct serial_val; struct serial_val;
struct key_pool; struct key_pool;
void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, std::string name, std::string type); void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, std::string name, std::string type);
void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, const char *arg, char **argv); void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, const char *arg, char **argv);
void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<serial_val> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state, key_pool &key_pool); void preserve_attribute(attribute_op const &op, std::string const &key, serial_val const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<serial_val> &full_values, key_pool &key_pool);
void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<mvt_value> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state, key_pool &key_pool); void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector<std::shared_ptr<std::string>> &full_keys, std::vector<mvt_value> &full_values, key_pool &key_pool);
extern std::map<std::string, attribute_op> numeric_operations; extern std::map<std::string, attribute_op> numeric_operations;
clip.cpp
+10 -12
View File
@@ -1371,13 +1371,12 @@ static void add_mean(mvt_feature &feature, mvt_layer &layer, std::string const &
}; };
// accumulate :sum:, :min:, :max:, and :count: versions of the specified attribute // accumulate :sum:, :min:, :max:, and :count: versions of the specified attribute
static void preserve_numeric(const std::string &key, const mvt_value &val, // numeric attribute being accumulated static void preserve_numeric(const std::string &key, const mvt_value &val, // numeric attribute being accumulated
std::vector<std::shared_ptr<std::string>> &full_keys, // keys of feature being accumulated onto std::vector<std::shared_ptr<std::string>> &full_keys, // keys of feature being accumulated onto
std::vector<mvt_value> &full_values, // values of features being accumulated onto std::vector<mvt_value> &full_values, // values of features being accumulated onto
const std::string &accumulate_numeric, // prefix of accumulations const std::string &accumulate_numeric, // prefix of accumulations
std::set<std::string> &keys, // key presence in the source feature std::set<std::string> &keys, // key presence in the source feature
std::map<std::string, size_t> &numeric_out_field, // key index in the output feature std::map<std::string, size_t> &numeric_out_field, // key index in the output feature
std::unordered_map<std::string, accum_state> &attribute_accum_state, // accumulation state for preserve_attribute()
key_pool &key_pool, key_pool &key_pool,
std::set<std::string> const &keep, std::set<std::string> const &exclude, std::set<std::string> const &keep, std::set<std::string> const &exclude,
std::vector<std::string> const &exclude_prefix) { std::vector<std::string> const &exclude_prefix) {
@@ -1457,7 +1456,7 @@ static void preserve_numeric(const std::string &key, const mvt_value &val,
full_values.push_back(v); full_values.push_back(v);
} else { } else {
full_values.push_back(full_values[out_attr->second]); full_values.push_back(full_values[out_attr->second]);
preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state, key_pool); preserve_attribute(op.second, prefixed, val, full_keys, full_values, key_pool);
} }
} }
} else { } else {
@@ -1470,7 +1469,7 @@ static void preserve_numeric(const std::string &key, const mvt_value &val,
full_values[prefixed_attr->second] = mvt_value(mvt_value_to_long_long(full_values[prefixed_attr->second]) + 1); full_values[prefixed_attr->second] = mvt_value(mvt_value_to_long_long(full_values[prefixed_attr->second]) + 1);
} }
} else { } else {
preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state, key_pool); preserve_attribute(op.second, prefixed, val, full_keys, full_values, key_pool);
} }
} }
} }
@@ -1597,7 +1596,6 @@ static bool feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
// attributes from the other features of the // attributes from the other features of the
// multiplier cluster accumulated onto them // multiplier cluster accumulated onto them
std::unordered_map<std::string, accum_state> attribute_accum_state;
std::vector<std::shared_ptr<std::string>> full_keys; std::vector<std::shared_ptr<std::string>> full_keys;
std::vector<mvt_value> full_values; std::vector<mvt_value> full_values;
std::map<std::string, size_t> numeric_out_field; std::map<std::string, size_t> numeric_out_field;
@@ -1642,13 +1640,13 @@ static bool feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
auto found = attribute_accum.find(key); auto found = attribute_accum.find(key);
if (found != attribute_accum.end()) { if (found != attribute_accum.end()) {
mvt_value val = features[i].layer->values[features[i].tags[j + 1]]; mvt_value val = features[i].layer->values[features[i].tags[j + 1]];
preserve_attribute(found->second, key, val, full_keys, full_values, attribute_accum_state, key_pool); preserve_attribute(found->second, key, val, full_keys, full_values, key_pool);
} else if (accumulate_numeric.size() > 0) { } else if (accumulate_numeric.size() > 0) {
const mvt_value &val = features[i].layer->values[features[i].tags[j + 1]]; const mvt_value &val = features[i].layer->values[features[i].tags[j + 1]];
if (val.is_numeric()) { if (val.is_numeric()) {
preserve_numeric(key, val, full_keys, full_values, preserve_numeric(key, val, full_keys, full_values,
accumulate_numeric, accumulate_numeric,
keys, numeric_out_field, attribute_accum_state, key_pool, keys, numeric_out_field, key_pool,
keep, exclude, exclude_prefix); keep, exclude, exclude_prefix);
} }
} }
mvt.hpp
+14 -1
View File
@@ -81,7 +81,8 @@ struct mvt_value;
double mvt_value_to_double(mvt_value const &v); double mvt_value_to_double(mvt_value const &v);
struct mvt_value { struct mvt_value {
mvt_value_type type; long /* mvt_value_type */ type : 5;
long count : 64 - 5;
std::shared_ptr<std::string> s; std::shared_ptr<std::string> s;
union { union {
@@ -138,6 +139,10 @@ struct mvt_value {
return mvt_value_to_double(*this); return mvt_value_to_double(*this);
} }
size_t get_count() const {
return count;
}
bool operator<(const mvt_value &o) const; bool operator<(const mvt_value &o) const;
bool operator==(const mvt_value &o) const; bool operator==(const mvt_value &o) const;
std::string toString() const; std::string toString() const;
@@ -145,11 +150,19 @@ struct mvt_value {
mvt_value() { mvt_value() {
this->type = mvt_double; this->type = mvt_double;
this->numeric_value.double_value = 0; this->numeric_value.double_value = 0;
this->count = 0;
} }
mvt_value(double v) { mvt_value(double v) {
this->type = mvt_double; this->type = mvt_double;
this->numeric_value.double_value = v; this->numeric_value.double_value = v;
this->count = 0;
}
void set_double_count(double v, size_t c) {
this->type = mvt_double;
this->numeric_value.double_value = v;
this->count = c;
} }
}; };
serial.hpp
+14 -1
View File
@@ -70,6 +70,20 @@ struct serial_val {
type = mvt_string; type = mvt_string;
s = val; s = val;
} }
size_t get_count() const {
size_t found = s.find('\0');
if (found == std::string::npos) {
return 0;
} else {
return atoll(s.c_str() + found + 1);
}
}
void set_double_count(double v, size_t c) {
type = mvt_double;
s = milo::dtoa_milo(v) + '\0' + std::to_string(c);
}
}; };
struct key_pool { struct key_pool {
@@ -152,7 +166,6 @@ struct serial_feature {
const char *stringpool; // string pool for keys/values lookup const char *stringpool; // string pool for keys/values lookup
std::shared_ptr<std::string> tile_stringpool; // string pool for mvt_value construction std::shared_ptr<std::string> tile_stringpool; // string pool for mvt_value construction
std::set<std::string> need_tilestats; std::set<std::string> need_tilestats;
std::unordered_map<std::string, accum_state> attribute_accum_state;
int z; // tile being produced int z; // tile being produced
int tx; int tx;
shared_borders.cpp
+47 -47
View File
@@ -83,13 +83,13 @@ bool edges_same(std::pair<std::vector<edge>::iterator, std::vector<edge>::iterat
return true; return true;
} }
bool find_common_edges(std::vector<serial_feature> &features, int z, int line_detail, double simplification, int maxzoom, double merge_fraction) { bool find_common_edges(std::vector<std::shared_ptr<serial_feature>> &features, int z, int line_detail, double simplification, int maxzoom, double merge_fraction) {
size_t merge_count = ceil((1 - merge_fraction) * features.size()); size_t merge_count = ceil((1 - merge_fraction) * features.size());
for (size_t i = 0; i < features.size(); i++) { for (size_t i = 0; i < features.size(); i++) {
if (features[i].t == VT_POLYGON) { if (features[i]->t == VT_POLYGON) {
{ {
drawvec &g = features[i].geometry; drawvec &g = features[i]->geometry;
drawvec out; drawvec out;
for (size_t k = 0; k < g.size(); k++) { for (size_t k = 0; k < g.size(); k++) {
@@ -100,7 +100,7 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
} }
} }
features[i].geometry = out; features[i]->geometry = out;
} }
} }
} }
@@ -112,21 +112,21 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
std::vector<edge> edges; std::vector<edge> edges;
size_t ring = 0; size_t ring = 0;
for (size_t i = 0; i < features.size(); i++) { for (size_t i = 0; i < features.size(); i++) {
if (features[i].t == VT_POLYGON) { if (features[i]->t == VT_POLYGON) {
{ {
for (size_t k = 0; k + 1 < features[i].geometry.size(); k++) { for (size_t k = 0; k + 1 < features[i]->geometry.size(); k++) {
if (features[i].geometry[k].op == VT_MOVETO) { if (features[i]->geometry[k].op == VT_MOVETO) {
ring++; ring++;
} }
if (features[i].geometry[k + 1].op == VT_LINETO) { if (features[i]->geometry[k + 1].op == VT_LINETO) {
drawvec dv; drawvec dv;
if (features[i].geometry[k] < features[i].geometry[k + 1]) { if (features[i]->geometry[k] < features[i]->geometry[k + 1]) {
dv.push_back(features[i].geometry[k]); dv.push_back(features[i]->geometry[k]);
dv.push_back(features[i].geometry[k + 1]); dv.push_back(features[i]->geometry[k + 1]);
} else { } else {
dv.push_back(features[i].geometry[k + 1]); dv.push_back(features[i]->geometry[k + 1]);
dv.push_back(features[i].geometry[k]); dv.push_back(features[i]->geometry[k]);
} }
edges.push_back(edge(dv[0].x, dv[0].y, dv[1].x, dv[1].y, ring)); edges.push_back(edge(dv[0].x, dv[0].y, dv[1].x, dv[1].y, ring));
@@ -143,9 +143,9 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
// is not the same as the set of rings using the edge on the other side. // is not the same as the set of rings using the edge on the other side.
for (size_t i = 0; i < features.size(); i++) { for (size_t i = 0; i < features.size(); i++) {
if (features[i].t == VT_POLYGON) { if (features[i]->t == VT_POLYGON) {
{ {
drawvec &g = features[i].geometry; drawvec &g = features[i]->geometry;
for (size_t k = 0; k < g.size(); k++) { for (size_t k = 0; k < g.size(); k++) {
g[k].necessary = 0; g[k].necessary = 0;
@@ -223,9 +223,9 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
// Roll rings that include a necessary point around so they start at one // Roll rings that include a necessary point around so they start at one
for (size_t i = 0; i < features.size(); i++) { for (size_t i = 0; i < features.size(); i++) {
if (features[i].t == VT_POLYGON) { if (features[i]->t == VT_POLYGON) {
{ {
drawvec &g = features[i].geometry; drawvec &g = features[i]->geometry;
for (size_t k = 0; k < g.size(); k++) { for (size_t k = 0; k < g.size(); k++) {
if (necessaries.count(g[k]) != 0) { if (necessaries.count(g[k]) != 0) {
@@ -315,21 +315,21 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
// Add new arc // Add new arc
size_t added = arcs.size() + 1; size_t added = arcs.size() + 1;
arcs.insert(std::pair<drawvec, size_t>(arc, added)); arcs.insert(std::pair<drawvec, size_t>(arc, added));
features[i].arc_polygon.push_back(added); features[i]->arc_polygon.push_back(added);
merge_candidates.insert(std::pair<ssize_t, size_t>(added, i)); merge_candidates.insert(std::pair<ssize_t, size_t>(added, i));
} else { } else {
features[i].arc_polygon.push_back(-(ssize_t) f2->second); features[i]->arc_polygon.push_back(-(ssize_t) f2->second);
merge_candidates.insert(std::pair<ssize_t, size_t>(-(ssize_t) f2->second, i)); merge_candidates.insert(std::pair<ssize_t, size_t>(-(ssize_t) f2->second, i));
} }
} else { } else {
features[i].arc_polygon.push_back(f->second); features[i]->arc_polygon.push_back(f->second);
merge_candidates.insert(std::pair<ssize_t, size_t>(f->second, i)); merge_candidates.insert(std::pair<ssize_t, size_t>(f->second, i));
} }
m = n - 1; m = n - 1;
} }
features[i].arc_polygon.push_back(0); features[i]->arc_polygon.push_back(0);
k = l - 1; k = l - 1;
} }
@@ -385,10 +385,10 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
if (r1i->second != r2i->second) { if (r1i->second != r2i->second) {
merge_order mo; merge_order mo;
mo.edge = i; mo.edge = i;
if (features[r1i->second].index > features[r2i->second].index) { if (features[r1i->second]->index > features[r2i->second]->index) {
mo.gap = features[r1i->second].index - features[r2i->second].index; mo.gap = features[r1i->second]->index - features[r2i->second]->index;
} else { } else {
mo.gap = features[r2i->second].index - features[r1i->second].index; mo.gap = features[r2i->second]->index - features[r1i->second]->index;
} }
mo.p1 = r1i->second; mo.p1 = r1i->second;
mo.p2 = r2i->second; mo.p2 = r2i->second;
@@ -406,19 +406,19 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
} }
size_t i = order[o].p1; size_t i = order[o].p1;
while (features[i].renamed >= 0) { while (features[i]->renamed >= 0) {
i = features[i].renamed; i = features[i]->renamed;
} }
size_t i2 = order[o].p2; size_t i2 = order[o].p2;
while (features[i2].renamed >= 0) { while (features[i2]->renamed >= 0) {
i2 = features[i2].renamed; i2 = features[i2]->renamed;
} }
for (size_t j = 0; j < features[i].arc_polygon.size() && merged < merge_count; j++) { for (size_t j = 0; j < features[i]->arc_polygon.size() && merged < merge_count; j++) {
if (features[i].arc_polygon[j] == order[o].edge) { if (features[i]->arc_polygon[j] == order[o].edge) {
{ {
// XXX snap links // XXX snap links
if (features[order[o].p2].arc_polygon.size() > 0) { if (features[order[o].p2]->arc_polygon.size() > 0) {
// This has to merge the ring that contains the anti-arc to this arc // This has to merge the ring that contains the anti-arc to this arc
// into the current ring, and then add whatever other rings were in // into the current ring, and then add whatever other rings were in
// that feature on to the end. // that feature on to the end.
@@ -427,11 +427,11 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
// the rings in order, but Wagyu should sort that out later // the rings in order, but Wagyu should sort that out later
std::vector<ssize_t> additions; std::vector<ssize_t> additions;
std::vector<ssize_t> &here = features[i].arc_polygon; std::vector<ssize_t> &here = features[i]->arc_polygon;
std::vector<ssize_t> &other = features[i2].arc_polygon; std::vector<ssize_t> &other = features[i2]->arc_polygon;
#if 0 #if 0
printf("seeking %zd\n", features[i].arc_polygon[j]); printf("seeking %zd\n", features[i]->arc_polygon[j]);
printf("before: "); printf("before: ");
for (size_t k = 0; k < here.size(); k++) { for (size_t k = 0; k < here.size(); k++) {
printf("%zd ", here[k]); printf("%zd ", here[k]);
@@ -464,7 +464,7 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
size_t m; size_t m;
for (m = k; m <= l; m++) { for (m = k; m <= l; m++) {
if (other[m] == -features[i].arc_polygon[j]) { if (other[m] == -features[i]->arc_polygon[j]) {
break; break;
} }
} }
@@ -494,12 +494,12 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
k = l; k = l;
} }
features[i2].arc_polygon.clear(); features[i2]->arc_polygon.clear();
features[i2].renamed = i; features[i2]->renamed = i;
merged++; merged++;
for (size_t k = 0; k < additions.size(); k++) { for (size_t k = 0; k < additions.size(); k++) {
features[i].arc_polygon.push_back(additions[k]); features[i]->arc_polygon.push_back(additions[k]);
} }
#if 0 #if 0
@@ -528,37 +528,37 @@ bool find_common_edges(std::vector<serial_feature> &features, int z, int line_de
// Turn the arc representations of the polygons back into standard polygon geometries // Turn the arc representations of the polygons back into standard polygon geometries
for (size_t i = 0; i < features.size(); i++) { for (size_t i = 0; i < features.size(); i++) {
if (features[i].t == VT_POLYGON) { if (features[i]->t == VT_POLYGON) {
features[i].geometry.clear(); features[i]->geometry.clear();
bool at_start = true; bool at_start = true;
draw first(-1, 0, 0); draw first(-1, 0, 0);
for (size_t j = 0; j < features[i].arc_polygon.size(); j++) { for (size_t j = 0; j < features[i]->arc_polygon.size(); j++) {
ssize_t p = features[i].arc_polygon[j]; ssize_t p = features[i]->arc_polygon[j];
if (p == 0) { if (p == 0) {
if (first.op >= 0) { if (first.op >= 0) {
features[i].geometry.push_back(first); features[i]->geometry.push_back(first);
first = draw(-1, 0, 0); first = draw(-1, 0, 0);
} }
at_start = true; at_start = true;
} else if (p > 0) { } else if (p > 0) {
for (size_t k = 0; k + 1 < simplified_arcs[p].size(); k++) { for (size_t k = 0; k + 1 < simplified_arcs[p].size(); k++) {
if (at_start) { if (at_start) {
features[i].geometry.push_back(draw(VT_MOVETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y)); features[i]->geometry.push_back(draw(VT_MOVETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y));
first = draw(VT_LINETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y); first = draw(VT_LINETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y);
} else { } else {
features[i].geometry.push_back(draw(VT_LINETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y)); features[i]->geometry.push_back(draw(VT_LINETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y));
} }
at_start = 0; at_start = 0;
} }
} else { /* p < 0 */ } else { /* p < 0 */
for (ssize_t k = simplified_arcs[-p].size() - 1; k > 0; k--) { for (ssize_t k = simplified_arcs[-p].size() - 1; k > 0; k--) {
if (at_start) { if (at_start) {
features[i].geometry.push_back(draw(VT_MOVETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y)); features[i]->geometry.push_back(draw(VT_MOVETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y));
first = draw(VT_LINETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y); first = draw(VT_LINETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y);
} else { } else {
features[i].geometry.push_back(draw(VT_LINETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y)); features[i]->geometry.push_back(draw(VT_LINETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y));
} }
at_start = 0; at_start = 0;
} }
shared_borders.hpp
+1 -1
View File
@@ -1 +1 @@
bool find_common_edges(std::vector<serial_feature> &features, int z, int line_detail, double simplification, int maxzoom, double merge_fraction); bool find_common_edges(std::vector<std::shared_ptr<serial_feature>> &features, int z, int line_detail, double simplification, int maxzoom, double merge_fraction);
tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json
+2 -2
View File
File diff suppressed because one or more lines are too long
tile.cpp
+131 -122
View File
@@ -92,13 +92,17 @@ static bool draws_something(drawvec const &geom) {
// comparator for --preserve-input-order, to reorder features back to their original input sequence // comparator for --preserve-input-order, to reorder features back to their original input sequence
static struct preservecmp { static struct preservecmp {
bool operator()(const std::vector<serial_feature> &a, const std::vector<serial_feature> &b) { bool operator()(const std::vector<std::shared_ptr<serial_feature>> &a, const std::vector<std::shared_ptr<serial_feature>> &b) {
return operator()(a[0], b[0]); return operator()(a[0], b[0]);
} }
bool operator()(const serial_feature &a, const serial_feature &b) { bool operator()(const serial_feature &a, const serial_feature &b) {
return a.seq < b.seq; return a.seq < b.seq;
} }
bool operator()(const std::shared_ptr<serial_feature> &a, const std::shared_ptr<serial_feature> &b) {
return a->seq < b->seq;
}
} preservecmp; } preservecmp;
static int metacmp(const serial_feature &one, const serial_feature &two); static int metacmp(const serial_feature &one, const serial_feature &two);
@@ -190,8 +194,8 @@ struct coalindexcmp_comparator {
return cmp; return cmp;
} }
bool operator()(const serial_feature &a, const serial_feature &o) const { bool operator()(const std::shared_ptr<serial_feature> &a, const std::shared_ptr<serial_feature> &o) const {
int cmp = coalindexcmp(&a, &o); int cmp = coalindexcmp(&*a, &*o);
if (cmp < 0) { if (cmp < 0) {
return true; return true;
} else { } else {
@@ -203,9 +207,9 @@ struct coalindexcmp_comparator {
static unsigned long long calculate_drop_sequence(serial_feature const &sf); static unsigned long long calculate_drop_sequence(serial_feature const &sf);
struct drop_sequence_cmp { struct drop_sequence_cmp {
bool operator()(const serial_feature &a, const serial_feature &b) { bool operator()(const std::shared_ptr<serial_feature> &a, const std::shared_ptr<serial_feature> &b) {
unsigned long long a_seq = calculate_drop_sequence(a); unsigned long long a_seq = calculate_drop_sequence(*a);
unsigned long long b_seq = calculate_drop_sequence(b); unsigned long long b_seq = calculate_drop_sequence(*b);
// sorts backwards, to put the features that would be dropped last, first here // sorts backwards, to put the features that would be dropped last, first here
if (a_seq > b_seq) { if (a_seq > b_seq) {
@@ -336,14 +340,14 @@ static mvt_value coerce_double(mvt_value v) {
// compare features numerically according to that sort key until the keys are exhausted. // compare features numerically according to that sort key until the keys are exhausted.
// If there is a tie, the feature with the earlier index (centroid) comes first. // If there is a tie, the feature with the earlier index (centroid) comes first.
struct ordercmp { struct ordercmp {
bool operator()(const std::vector<serial_feature> &a, const std::vector<serial_feature> &b) { bool operator()(const std::vector<std::shared_ptr<serial_feature>> &a, const std::vector<std::shared_ptr<serial_feature>> &b) {
return operator()(a[0], b[0]); return operator()(a[0], b[0]);
} }
bool operator()(const serial_feature &a, const serial_feature &b) { bool operator()(const std::shared_ptr<serial_feature> &a, const std::shared_ptr<serial_feature> &b) {
for (size_t i = 0; i < order_by.size(); i++) { for (size_t i = 0; i < order_by.size(); i++) {
mvt_value v1 = coerce_double(find_attribute_value(&a, order_by[i].name)); mvt_value v1 = coerce_double(find_attribute_value(&*a, order_by[i].name));
mvt_value v2 = coerce_double(find_attribute_value(&b, order_by[i].name)); mvt_value v2 = coerce_double(find_attribute_value(&*b, order_by[i].name));
if (order_by[i].descending) { if (order_by[i].descending) {
if (v2 < v1) { if (v2 < v1) {
@@ -360,7 +364,7 @@ struct ordercmp {
} }
} }
if (a.index < b.index) { if (a->index < b->index) {
return true; return true;
} }
@@ -370,12 +374,12 @@ struct ordercmp {
// For --retain-points-multiplier: Go through a list of features and return a list of clusters of features, // For --retain-points-multiplier: Go through a list of features and return a list of clusters of features,
// creating a new cluster whenever the tippecanoe:retain_points_multiplier_first attribute is seen. // creating a new cluster whenever the tippecanoe:retain_points_multiplier_first attribute is seen.
static std::vector<std::vector<serial_feature>> assemble_multiplier_clusters(std::vector<serial_feature> const &features) { static std::vector<std::vector<std::shared_ptr<serial_feature>>> assemble_multiplier_clusters(std::vector<std::shared_ptr<serial_feature>> const &features) {
std::vector<std::vector<serial_feature>> clusters; std::vector<std::vector<std::shared_ptr<serial_feature>>> clusters;
if (retain_points_multiplier == 1) { if (retain_points_multiplier == 1) {
for (auto const &feature : features) { for (auto const &feature : features) {
std::vector<serial_feature> cluster; std::vector<std::shared_ptr<serial_feature>> cluster;
cluster.push_back(std::move(feature)); cluster.push_back(std::move(feature));
clusters.push_back(std::move(cluster)); clusters.push_back(std::move(cluster));
} }
@@ -383,8 +387,8 @@ static std::vector<std::vector<serial_feature>> assemble_multiplier_clusters(std
for (auto const &feature : features) { for (auto const &feature : features) {
bool is_cluster_start = false; bool is_cluster_start = false;
for (size_t i = 0; i < feature.full_keys.size(); i++) { for (size_t i = 0; i < feature->full_keys.size(); i++) {
if (*feature.full_keys[i] == "tippecanoe:retain_points_multiplier_first") { if (*feature->full_keys[i] == "tippecanoe:retain_points_multiplier_first") {
is_cluster_start = true; is_cluster_start = true;
break; break;
} }
@@ -404,20 +408,20 @@ static std::vector<std::vector<serial_feature>> assemble_multiplier_clusters(std
// For --retain-points-multiplier: Flatten a list of clusters of features back into a list of features, // For --retain-points-multiplier: Flatten a list of clusters of features back into a list of features,
// moving the "tippecanoe:retain_points_multiplier_first" attribute onto the first feature of each cluster // moving the "tippecanoe:retain_points_multiplier_first" attribute onto the first feature of each cluster
// if it is not already there. // if it is not already there.
static std::vector<serial_feature> disassemble_multiplier_clusters(std::vector<std::vector<serial_feature>> &clusters) { static std::vector<std::shared_ptr<serial_feature>> disassemble_multiplier_clusters(std::vector<std::vector<std::shared_ptr<serial_feature>>> &clusters) {
std::vector<serial_feature> out; std::vector<std::shared_ptr<serial_feature>> out;
for (auto &cluster : clusters) { for (auto &cluster : clusters) {
// fix up the attributes so the first feature of the multiplier cluster // fix up the attributes so the first feature of the multiplier cluster
// gets the marker attribute // gets the marker attribute
for (size_t i = 0; i < cluster.size(); i++) { for (size_t i = 0; i < cluster.size(); i++) {
for (size_t j = 0; j < cluster[i].full_keys.size(); j++) { for (size_t j = 0; j < cluster[i]->full_keys.size(); j++) {
if (*cluster[i].full_keys[j] == "tippecanoe:retain_points_multiplier_first") { if (*cluster[i]->full_keys[j] == "tippecanoe:retain_points_multiplier_first") {
cluster[0].full_keys.push_back(std::move(cluster[i].full_keys[j])); cluster[0]->full_keys.push_back(std::move(cluster[i]->full_keys[j]));
cluster[0].full_values.push_back(std::move(cluster[i].full_values[j])); cluster[0]->full_values.push_back(std::move(cluster[i]->full_values[j]));
cluster[i].full_keys.erase(cluster[i].full_keys.begin() + j); cluster[i]->full_keys.erase(cluster[i]->full_keys.begin() + j);
cluster[i].full_values.erase(cluster[i].full_values.begin() + j); cluster[i]->full_values.erase(cluster[i]->full_values.begin() + j);
i = cluster.size(); // break outer i = cluster.size(); // break outer
break; break;
@@ -534,7 +538,7 @@ static void rewrite(serial_feature const &osf, int z, int nextzoom, int maxzoom,
// This is the parameter block passed to each simplification worker thread // This is the parameter block passed to each simplification worker thread
struct simplification_worker_arg { struct simplification_worker_arg {
std::vector<serial_feature> *features = NULL; std::vector<std::shared_ptr<serial_feature>> *features = NULL;
int task = 0; int task = 0;
int tasks = 0; int tasks = 0;
bool trying_to_stop_early = false; bool trying_to_stop_early = false;
@@ -668,19 +672,19 @@ static double simplify_feature(serial_feature *p, drawvec const &shared_nodes, n
// simplify and clean the geometry of batches of features. // simplify and clean the geometry of batches of features.
static void *simplification_worker(void *v) { static void *simplification_worker(void *v) {
simplification_worker_arg *a = (simplification_worker_arg *) v; simplification_worker_arg *a = (simplification_worker_arg *) v;
std::vector<serial_feature> *features = a->features; std::vector<std::shared_ptr<serial_feature>> *features = a->features;
for (size_t i = a->task; i < (*features).size(); i += a->tasks) { for (size_t i = a->task; i < (*features).size(); i += a->tasks) {
double area = 0; double area = 0;
if (!a->trying_to_stop_early) { if (!a->trying_to_stop_early) {
area = simplify_feature(&((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos, *(a->shared_nodes_bloom)); area = simplify_feature(&*((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos, *(a->shared_nodes_bloom));
} }
signed char t = (*features)[i].t; signed char t = (*features)[i]->t;
int z = (*features)[i].z; int z = (*features)[i]->z;
int out_detail = (*features)[i].extra_detail; int out_detail = (*features)[i]->extra_detail;
drawvec geom = (*features)[i].geometry; drawvec geom = (*features)[i]->geometry;
to_tile_scale(geom, z, out_detail); to_tile_scale(geom, z, out_detail);
if (t == VT_POLYGON) { if (t == VT_POLYGON) {
@@ -709,15 +713,15 @@ static void *simplification_worker(void *v) {
} }
if (t == VT_POLYGON && additional[A_GENERATE_POLYGON_LABEL_POINTS]) { if (t == VT_POLYGON && additional[A_GENERATE_POLYGON_LABEL_POINTS]) {
t = (*features)[i].t = VT_POINT; t = (*features)[i]->t = VT_POINT;
geom = checkerboard_anchors(from_tile_scale(geom, z, out_detail), (*features)[i].tx, (*features)[i].ty, z, (*features)[i].label_point); geom = checkerboard_anchors(from_tile_scale(geom, z, out_detail), (*features)[i]->tx, (*features)[i]->ty, z, (*features)[i]->label_point);
to_tile_scale(geom, z, out_detail); to_tile_scale(geom, z, out_detail);
} }
if ((*features)[i].index == 0) { if ((*features)[i]->index == 0) {
(*features)[i].index = i; (*features)[i]->index = i;
} }
(*features)[i].geometry = std::move(geom); (*features)[i]->geometry = std::move(geom);
} }
return NULL; return NULL;
@@ -1500,7 +1504,7 @@ void preserve_attributes(std::unordered_map<std::string, attribute_op> const *at
sv.s = sf.stringpool + sf.values[i] + 1; sv.s = sf.stringpool + sf.values[i] + 1;
promote_attribute(key, p, key_pool); promote_attribute(key, p, key_pool);
preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state, key_pool); preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, key_pool);
} else if (type == mvt_double && accumulate_numeric.size() > 0 && !starts_with(key, accumulate_numeric_colon)) { } else if (type == mvt_double && accumulate_numeric.size() > 0 && !starts_with(key, accumulate_numeric_colon)) {
for (auto const &operation : numeric_operations) { for (auto const &operation : numeric_operations) {
serial_val sv; serial_val sv;
@@ -1509,7 +1513,7 @@ void preserve_attributes(std::unordered_map<std::string, attribute_op> const *at
std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key; std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key;
promote_attribute_prefix(key, prefixed_key, p, key_pool); promote_attribute_prefix(key, prefixed_key, p, key_pool);
preserve_attribute(operation.second, prefixed_key, sv, p.full_keys, p.full_values, p.attribute_accum_state, key_pool); preserve_attribute(operation.second, prefixed_key, sv, p.full_keys, p.full_values, key_pool);
} }
} }
} }
@@ -1522,12 +1526,12 @@ void preserve_attributes(std::unordered_map<std::string, attribute_op> const *at
const serial_val &sv = sf.full_values[i]; const serial_val &sv = sf.full_values[i];
promote_attribute(key, p, key_pool); // promotes it in the target feature promote_attribute(key, p, key_pool); // promotes it in the target feature
preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state, key_pool); preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, key_pool);
} else if (type == mvt_double && accumulate_numeric.size() > 0 && !starts_with(key, accumulate_numeric_colon)) { } else if (type == mvt_double && accumulate_numeric.size() > 0 && !starts_with(key, accumulate_numeric_colon)) {
for (auto const &operation : numeric_operations) { for (auto const &operation : numeric_operations) {
std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key; std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key;
promote_attribute_prefix(key, prefixed_key, p, key_pool); promote_attribute_prefix(key, prefixed_key, p, key_pool);
preserve_attribute(operation.second, prefixed_key, sf.full_values[i], p.full_keys, p.full_values, p.attribute_accum_state, key_pool); preserve_attribute(operation.second, prefixed_key, sf.full_values[i], p.full_keys, p.full_values, key_pool);
} }
} }
} }
@@ -1538,13 +1542,13 @@ void preserve_attributes(std::unordered_map<std::string, attribute_op> const *at
// ordinarily returns the most recently-added feature from the same layer as the feature // ordinarily returns the most recently-added feature from the same layer as the feature
// that is being dropped. // that is being dropped.
// //
bool find_feature_to_accumulate_onto(std::vector<serial_feature> &features, serial_feature &sf, ssize_t &out, std::vector<std::vector<std::string>> *layer_unmaps, long long maxextent) { bool find_feature_to_accumulate_onto(std::vector<std::shared_ptr<serial_feature>> &features, serial_feature &sf, ssize_t &out, std::vector<std::vector<std::string>> *layer_unmaps, long long maxextent) {
for (size_t i = features.size(); i > 0; i--) { for (size_t i = features.size(); i > 0; i--) {
if (features[i - 1].t == sf.t) { if (features[i - 1]->t == sf.t) {
std::string &layername1 = (*layer_unmaps)[features[i - 1].segment][features[i - 1].layer]; std::string &layername1 = (*layer_unmaps)[features[i - 1]->segment][features[i - 1]->layer];
std::string &layername2 = (*layer_unmaps)[sf.segment][sf.layer]; std::string &layername2 = (*layer_unmaps)[sf.segment][sf.layer];
if (layername1 == layername2 && features[i - 1].extent <= maxextent) { if (layername1 == layername2 && features[i - 1]->extent <= maxextent) {
out = i - 1; out = i - 1;
return true; return true;
} }
@@ -1617,8 +1621,8 @@ return;
// This is the structure that the features from each layer are accumulated into // This is the structure that the features from each layer are accumulated into
struct layer_features { struct layer_features {
std::vector<serial_feature> features; // The features of this layer, so far std::vector<std::shared_ptr<serial_feature>> features; // The features of this layer, so far
size_t multiplier_cluster_size = 0; // The feature count of the current multiplier cluster size_t multiplier_cluster_size = 0; // The feature count of the current multiplier cluster
}; };
bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features &layer, serial_feature &sf, std::vector<std::vector<std::string>> *layer_unmaps, strategy &strategy, bool &drop_rest, std::unordered_map<std::string, attribute_op> const *attribute_accum, key_pool &key_pool) { bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features &layer, serial_feature &sf, std::vector<std::vector<std::string>> *layer_unmaps, strategy &strategy, bool &drop_rest, std::unordered_map<std::string, attribute_op> const *attribute_accum, key_pool &key_pool) {
@@ -1632,7 +1636,7 @@ bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features
sf.dropped = layer.multiplier_cluster_size + 1; sf.dropped = layer.multiplier_cluster_size + 1;
return false; // converted rather than dropped return false; // converted rather than dropped
} else { } else {
preserve_attributes(attribute_accum, sf, layer.features[which_serial_feature], key_pool); preserve_attributes(attribute_accum, sf, *layer.features[which_serial_feature], key_pool);
drop_rest = true; drop_rest = true;
return true; // dropped return true; // dropped
} }
@@ -1872,7 +1876,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
layers.emplace(layername, layer_features()); layers.emplace(layername, layer_features());
} }
struct layer_features &layer = layers.find(layername)->second; struct layer_features &layer = layers.find(layername)->second;
std::vector<serial_feature> &features = layer.features; std::vector<std::shared_ptr<serial_feature>> &features = layer.features;
if (sf.t == VT_POINT) { if (sf.t == VT_POINT) {
if (extent_previndex >= sf.index) { if (extent_previndex >= sf.index) {
@@ -1921,7 +1925,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (sf.dropped == FEATURE_DROPPED || drop_rest) { if (sf.dropped == FEATURE_DROPPED || drop_rest) {
if (find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { if (find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) {
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
strategy.dropped_by_rate++; strategy.dropped_by_rate++;
can_stop_early = false; can_stop_early = false;
continue; continue;
@@ -1934,7 +1938,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (sf.dropped == FEATURE_KEPT) { if (sf.dropped == FEATURE_KEPT) {
if (gamma > 0) { if (gamma > 0) {
if (manage_gap(sf.index, &previndex, scale, gamma, &gap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { if (manage_gap(sf.index, &previndex, scale, gamma, &gap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) {
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
strategy.dropped_by_gamma++; strategy.dropped_by_gamma++;
drop_rest = true; drop_rest = true;
can_stop_early = false; can_stop_early = false;
@@ -1949,20 +1953,20 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// rather than wanting each feature to have a consistent // rather than wanting each feature to have a consistent
// idea of density between zooms. // idea of density between zooms.
if ((sf.index < merge_previndex || sf.index - merge_previndex < cluster_mingap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { if ((sf.index < merge_previndex || sf.index - merge_previndex < cluster_mingap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) {
features[which_serial_feature].clustered++; features[which_serial_feature]->clustered++;
if (features[which_serial_feature].t == VT_POINT && if (features[which_serial_feature]->t == VT_POINT &&
features[which_serial_feature].geometry.size() == 1 && features[which_serial_feature]->geometry.size() == 1 &&
sf.geometry.size() == 1) { sf.geometry.size() == 1) {
double x = (double) features[which_serial_feature].geometry[0].x * features[which_serial_feature].clustered; double x = (double) features[which_serial_feature]->geometry[0].x * features[which_serial_feature]->clustered;
double y = (double) features[which_serial_feature].geometry[0].y * features[which_serial_feature].clustered; double y = (double) features[which_serial_feature]->geometry[0].y * features[which_serial_feature]->clustered;
x += sf.geometry[0].x; x += sf.geometry[0].x;
y += sf.geometry[0].y; y += sf.geometry[0].y;
features[which_serial_feature].geometry[0].x = x / (features[which_serial_feature].clustered + 1); features[which_serial_feature]->geometry[0].x = x / (features[which_serial_feature]->clustered + 1);
features[which_serial_feature].geometry[0].y = y / (features[which_serial_feature].clustered + 1); features[which_serial_feature]->geometry[0].y = y / (features[which_serial_feature]->clustered + 1);
} }
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
strategy.coalesced_as_needed++; strategy.coalesced_as_needed++;
drop_rest = true; drop_rest = true;
can_stop_early = false; can_stop_early = false;
@@ -1981,20 +1985,20 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// it averages the point locations // it averages the point locations
add_sample_to(gaps, sf.gap, gaps_increment, seq); add_sample_to(gaps, sf.gap, gaps_increment, seq);
if (sf.gap < mingap && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { if (sf.gap < mingap && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) {
features[which_serial_feature].clustered++; features[which_serial_feature]->clustered++;
if (features[which_serial_feature].t == VT_POINT && if (features[which_serial_feature]->t == VT_POINT &&
features[which_serial_feature].geometry.size() == 1 && features[which_serial_feature]->geometry.size() == 1 &&
sf.geometry.size() == 1) { sf.geometry.size() == 1) {
double x = (double) features[which_serial_feature].geometry[0].x * features[which_serial_feature].clustered; double x = (double) features[which_serial_feature]->geometry[0].x * features[which_serial_feature]->clustered;
double y = (double) features[which_serial_feature].geometry[0].y * features[which_serial_feature].clustered; double y = (double) features[which_serial_feature]->geometry[0].y * features[which_serial_feature]->clustered;
x += sf.geometry[0].x; x += sf.geometry[0].x;
y += sf.geometry[0].y; y += sf.geometry[0].y;
features[which_serial_feature].geometry[0].x = x / (features[which_serial_feature].clustered + 1); features[which_serial_feature]->geometry[0].x = x / (features[which_serial_feature]->clustered + 1);
features[which_serial_feature].geometry[0].y = y / (features[which_serial_feature].clustered + 1); features[which_serial_feature]->geometry[0].y = y / (features[which_serial_feature]->clustered + 1);
} }
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
strategy.coalesced_as_needed++; strategy.coalesced_as_needed++;
drop_rest = true; drop_rest = true;
continue; continue;
@@ -2002,10 +2006,10 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
} else if (additional[A_COALESCE_DENSEST_AS_NEEDED]) { } else if (additional[A_COALESCE_DENSEST_AS_NEEDED]) {
add_sample_to(gaps, sf.gap, gaps_increment, seq); add_sample_to(gaps, sf.gap, gaps_increment, seq);
if (sf.gap < mingap && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { if (sf.gap < mingap && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) {
coalesce_geometry(features[which_serial_feature], sf); coalesce_geometry(*features[which_serial_feature], sf);
features[which_serial_feature].coalesced = true; features[which_serial_feature]->coalesced = true;
coalesced_area += sf.extent; coalesced_area += sf.extent;
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
strategy.coalesced_as_needed++; strategy.coalesced_as_needed++;
drop_rest = true; drop_rest = true;
can_stop_early = false; can_stop_early = false;
@@ -2024,10 +2028,10 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
} else if (additional[A_COALESCE_SMALLEST_AS_NEEDED]) { } else if (additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
add_sample_to(extents, sf.extent, extents_increment, seq); add_sample_to(extents, sf.extent, extents_increment, seq);
if (minextent != 0 && sf.extent + coalesced_area <= minextent && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, minextent)) { if (minextent != 0 && sf.extent + coalesced_area <= minextent && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, minextent)) {
coalesce_geometry(features[which_serial_feature], sf); coalesce_geometry(*features[which_serial_feature], sf);
features[which_serial_feature].coalesced = true; features[which_serial_feature]->coalesced = true;
coalesced_area += sf.extent; coalesced_area += sf.extent;
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
strategy.coalesced_as_needed++; strategy.coalesced_as_needed++;
drop_rest = true; drop_rest = true;
can_stop_early = false; can_stop_early = false;
@@ -2044,9 +2048,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
} else if (additional[A_COALESCE_FRACTION_AS_NEEDED]) { } else if (additional[A_COALESCE_FRACTION_AS_NEEDED]) {
add_sample_to(drop_sequences, drop_sequence, drop_sequences_increment, seq); add_sample_to(drop_sequences, drop_sequence, drop_sequences_increment, seq);
if (mindrop_sequence != 0 && drop_sequence <= mindrop_sequence && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) { if (mindrop_sequence != 0 && drop_sequence <= mindrop_sequence && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX)) {
coalesce_geometry(features[which_serial_feature], sf); coalesce_geometry(*features[which_serial_feature], sf);
features[which_serial_feature].coalesced = true; features[which_serial_feature]->coalesced = true;
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature], key_pool); preserve_attributes(arg->attribute_accum, sf, *features[which_serial_feature], key_pool);
strategy.coalesced_as_needed++; strategy.coalesced_as_needed++;
drop_rest = true; drop_rest = true;
can_stop_early = false; can_stop_early = false;
@@ -2188,23 +2192,23 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
tile_detail = sf.extra_detail; tile_detail = sf.extra_detail;
} }
features.push_back(std::move(sf)); features.push_back(std::make_shared<serial_feature>(sf));
unsimplified_geometry_size += features.back().geometry.size() * sizeof(draw); unsimplified_geometry_size += features.back()->geometry.size() * sizeof(draw);
if (unsimplified_geometry_size > 10 * 1024 * 1024 && !additional[A_DETECT_SHARED_BORDERS]) { if (unsimplified_geometry_size > 10 * 1024 * 1024 && !additional[A_DETECT_SHARED_BORDERS]) {
// we should be safe to simplify here with P_SIMPLIFY_SHARED_NODES, since they will // we should be safe to simplify here with P_SIMPLIFY_SHARED_NODES, since they will
// have been assembled globally, although that also means that simplification // have been assembled globally, although that also means that simplification
// may not be very effective for reducing memory usage. // may not be very effective for reducing memory usage.
for (; simplified_geometry_through < features.size(); simplified_geometry_through++) { for (; simplified_geometry_through < features.size(); simplified_geometry_through++) {
simplify_feature(&features[simplified_geometry_through], shared_nodes, shared_nodes_map, nodepos, shared_nodes_bloom); simplify_feature(&*features[simplified_geometry_through], shared_nodes, shared_nodes_map, nodepos, shared_nodes_bloom);
if (features[simplified_geometry_through].t == VT_POLYGON) { if (features[simplified_geometry_through]->t == VT_POLYGON) {
drawvec to_clean = features[simplified_geometry_through].geometry; drawvec to_clean = features[simplified_geometry_through]->geometry;
// don't scale up because this is still world coordinates // don't scale up because this is still world coordinates
to_clean = clean_or_clip_poly(to_clean, 0, 0, false, false); to_clean = clean_or_clip_poly(to_clean, 0, 0, false, false);
features[simplified_geometry_through].geometry = std::move(to_clean); features[simplified_geometry_through]->geometry = std::move(to_clean);
} }
} }
@@ -2293,7 +2297,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
for (auto &kv : layers) { for (auto &kv : layers) {
std::string const &layername = kv.first; std::string const &layername = kv.first;
std::vector<serial_feature> &features = kv.second.features; std::vector<std::shared_ptr<serial_feature>> &features = kv.second.features;
if (retain_points_multiplier > 1) { if (retain_points_multiplier > 1) {
add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "tippecanoe:retain_points_multiplier_first", serial_val(mvt_bool, "true")); add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "tippecanoe:retain_points_multiplier_first", serial_val(mvt_bool, "true"));
@@ -2302,7 +2306,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
std::vector<std::pair<size_t, size_t>> feature_sequences; std::vector<std::pair<size_t, size_t>> feature_sequences;
for (size_t i = 0; i < features.size(); i++) { for (size_t i = 0; i < features.size(); i++) {
feature_sequences.emplace_back(features[i].seq, i); feature_sequences.emplace_back(features[i]->seq, i);
} }
// tag each feature with its sequence number within the layer // tag each feature with its sequence number within the layer
@@ -2316,15 +2320,15 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
size_t j = feature_sequences[i].second; size_t j = feature_sequences[i].second;
serial_val sv(mvt_double, std::to_string(i)); serial_val sv(mvt_double, std::to_string(i));
features[j].full_keys.push_back(key_pool.pool("tippecanoe:retain_points_multiplier_sequence")); features[j]->full_keys.push_back(key_pool.pool("tippecanoe:retain_points_multiplier_sequence"));
features[j].full_values.push_back(sv); features[j]->full_values.push_back(sv);
add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *features[j].full_keys.back(), sv); add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *features[j]->full_keys.back(), sv);
} }
} }
for (size_t i = 0; i < features.size(); i++) { for (size_t i = 0; i < features.size(); i++) {
serial_feature &p = features[i]; serial_feature &p = *features[i];
if (p.clustered > 0) { if (p.clustered > 0) {
serial_val sv, sv2, sv3, sv4; serial_val sv, sv2, sv3, sv4;
@@ -2370,6 +2374,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (p.need_tilestats.size() > 0) { if (p.need_tilestats.size() > 0) {
for (size_t j = 0; j < p.full_keys.size(); j++) { for (size_t j = 0; j < p.full_keys.size(); j++) {
if (p.need_tilestats.count(*p.full_keys[j]) > 0) { if (p.need_tilestats.count(*p.full_keys[j]) > 0) {
// remove accumulation state
size_t found = p.full_values[j].s.find('\0');
if (found != std::string::npos) {
p.full_values[j].s = p.full_values[j].s.substr(0, found);
}
add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]); add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]);
} }
} }
@@ -2421,18 +2430,18 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
} }
for (size_t i = 0; i < features.size(); i++) { for (size_t i = 0; i < features.size(); i++) {
signed char t = features[i].t; signed char t = features[i]->t;
{ {
if (t == VT_POINT || draws_something(features[i].geometry)) { if (t == VT_POINT || draws_something(features[i]->geometry)) {
// printf("segment %d layer %lld is %s\n", features[i].segment, features[i].layer, (*layer_unmaps)[features[i].segment][features[i].layer].c_str()); // printf("segment %d layer %lld is %s\n", features[i].segment, features[i].layer, (*layer_unmaps)[features[i].segment][features[i].layer].c_str());
features[i].coalesced = false; features[i]->coalesced = false;
} }
} }
} }
std::vector<serial_feature> &layer_features = features; std::vector<std::shared_ptr<serial_feature>> &layer_features = features;
if (additional[A_REORDER]) { if (additional[A_REORDER]) {
std::stable_sort(layer_features.begin(), layer_features.end(), coalindexcmp_comparator()); std::stable_sort(layer_features.begin(), layer_features.end(), coalindexcmp_comparator());
@@ -2449,11 +2458,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
for (size_t x = 1; x < layer_features.size(); x++) { for (size_t x = 1; x < layer_features.size(); x++) {
size_t y = out - 1; size_t y = out - 1;
if (out > 0 && coalcmp(&layer_features[x], &layer_features[y]) == 0) { if (out > 0 && coalcmp(&*layer_features[x], &*layer_features[y]) == 0) {
for (size_t g = 0; g < layer_features[x].geometry.size(); g++) { for (size_t g = 0; g < layer_features[x]->geometry.size(); g++) {
layer_features[y].geometry.push_back(std::move(layer_features[x].geometry[g])); layer_features[y]->geometry.push_back(std::move(layer_features[x]->geometry[g]));
} }
layer_features[y].coalesced = true; layer_features[y]->coalesced = true;
} else { } else {
layer_features[out++] = layer_features[x]; layer_features[out++] = layer_features[x];
} }
@@ -2471,25 +2480,25 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
size_t out = 0; size_t out = 0;
for (size_t x = 0; x < layer_features.size(); x++) { for (size_t x = 0; x < layer_features.size(); x++) {
if (layer_features[x].coalesced && layer_features[x].t == VT_LINE) { if (layer_features[x]->coalesced && layer_features[x]->t == VT_LINE) {
layer_features[x].geometry = remove_noop(layer_features[x].geometry, layer_features[x].t, 0); layer_features[x]->geometry = remove_noop(layer_features[x]->geometry, layer_features[x]->t, 0);
if (!(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]))) { if (!(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]))) {
// XXX revisit: why does this not take zoom into account? // XXX revisit: why does this not take zoom into account?
layer_features[x].geometry = simplify_lines(layer_features[x].geometry, 32, 0, 0, 0, layer_features[x]->geometry = simplify_lines(layer_features[x]->geometry, 32, 0, 0, 0,
!(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, layer_features[x].t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, ""); !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, layer_features[x]->t == VT_POLYGON ? 4 : 0, shared_nodes, NULL, 0, "");
} }
} }
if (layer_features[x].t == VT_POLYGON) { if (layer_features[x]->t == VT_POLYGON) {
if (layer_features[x].coalesced) { if (layer_features[x]->coalesced) {
// we can try scaling up because this is tile coordinates // we can try scaling up because this is tile coordinates
layer_features[x].geometry = clean_or_clip_poly(layer_features[x].geometry, 0, 0, false, true); layer_features[x]->geometry = clean_or_clip_poly(layer_features[x]->geometry, 0, 0, false, true);
} }
layer_features[x].geometry = close_poly(layer_features[x].geometry); layer_features[x]->geometry = close_poly(layer_features[x]->geometry);
} }
if (layer_features[x].geometry.size() > 0) { if (layer_features[x]->geometry.size() > 0) {
layer_features[out++] = layer_features[x]; layer_features[out++] = layer_features[x];
} }
} }
@@ -2530,7 +2539,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
size_t feature_count = 0; size_t feature_count = 0;
for (auto layer_iterator = layers.begin(); layer_iterator != layers.end(); ++layer_iterator) { for (auto layer_iterator = layers.begin(); layer_iterator != layers.end(); ++layer_iterator) {
std::vector<serial_feature> &layer_features = layer_iterator->second.features; std::vector<std::shared_ptr<serial_feature>> &layer_features = layer_iterator->second.features;
feature_count += layer_features.size(); feature_count += layer_features.size();
mvt_layer layer; mvt_layer layer;
@@ -2541,34 +2550,34 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
for (size_t x = 0; x < layer_features.size(); x++) { for (size_t x = 0; x < layer_features.size(); x++) {
mvt_feature feature; mvt_feature feature;
if (layer_features[x].t == VT_LINE || layer_features[x].t == VT_POLYGON) { if (layer_features[x]->t == VT_LINE || layer_features[x]->t == VT_POLYGON) {
layer_features[x].geometry = remove_noop(layer_features[x].geometry, layer_features[x].t, 0); layer_features[x]->geometry = remove_noop(layer_features[x]->geometry, layer_features[x]->t, 0);
} }
if (layer_features[x].geometry.size() == 0) { if (layer_features[x]->geometry.size() == 0) {
layer_features[x] = serial_feature(); layer_features[x] = std::make_shared<serial_feature>();
continue; continue;
} }
feature.type = layer_features[x].t; feature.type = layer_features[x]->t;
feature.geometry = to_feature(layer_features[x].geometry); feature.geometry = to_feature(layer_features[x]->geometry);
count += layer_features[x].geometry.size(); count += layer_features[x]->geometry.size();
layer_features[x].geometry.clear(); layer_features[x]->geometry.clear();
feature.id = layer_features[x].id; feature.id = layer_features[x]->id;
feature.has_id = layer_features[x].has_id; feature.has_id = layer_features[x]->has_id;
decode_meta(layer_features[x], layer, feature); decode_meta(*layer_features[x], layer, feature);
for (size_t a = 0; a < layer_features[x].full_keys.size(); a++) { for (size_t a = 0; a < layer_features[x]->full_keys.size(); a++) {
serial_val sv = layer_features[x].full_values[a]; serial_val sv = layer_features[x]->full_values[a];
mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool);
layer.tag(feature, *layer_features[x].full_keys[a], v); layer.tag(feature, *layer_features[x]->full_keys[a], v);
} }
if (additional[A_CALCULATE_FEATURE_DENSITY]) { if (additional[A_CALCULATE_FEATURE_DENSITY]) {
int glow = 255; int glow = 255;
if (layer_features[x].spacing > 0) { if (layer_features[x]->spacing > 0) {
glow = (1 / layer_features[x].spacing); glow = (1 / layer_features[x]->spacing);
if (glow > 255) { if (glow > 255) {
glow = 255; glow = 255;
} }
@@ -2587,7 +2596,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
} }
layer.features.push_back(std::move(feature)); layer.features.push_back(std::move(feature));
layer_features[x] = serial_feature(); layer_features[x] = std::make_shared<serial_feature>();
} }
if (layer.features.size() > 0) { if (layer.features.size() > 0) {
version.hpp
+1 -1
View File
@@ -1,6 +1,6 @@
#ifndef VERSION_HPP #ifndef VERSION_HPP
#define VERSION_HPP #define VERSION_HPP
#define VERSION "v2.74.0" #define VERSION "v2.75.0"
#endif #endif