diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Improve the memory spike during tile construction

Browse Source
This commit is contained in:
Erica Fischer
2025-01-31 09:00:43 -08:00
parent 583fc3744a
commit c2e8da0881
2 changed files with 20 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mvt.cpp
+17 -9
View File
@@ -316,6 +316,10 @@ struct sorted_value {
return false; return false;
} }
bool operator()(const std::shared_ptr<sorted_value> &a, const std::shared_ptr<sorted_value> &b) {
return *a < *b;
}
}; };
std::string mvt_tile::encode() { std::string mvt_tile::encode() {
@@ -334,7 +338,7 @@ std::string mvt_tile::encode() {
layer_writer.add_string(3, layers[i].keys[j]); /* key */ layer_writer.add_string(3, layers[i].keys[j]); /* key */
} }
std::vector<sorted_value> sorted_values; std::vector<std::shared_ptr<sorted_value>> sorted_values;
for (size_t v = 0; v < layers[i].values.size(); v++) { for (size_t v = 0; v < layers[i].values.size(); v++) {
std::string value_string; std::string value_string;
@@ -371,30 +375,34 @@ std::string mvt_tile::encode() {
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
sorted_value sv; std::shared_ptr<sorted_value> sv = std::make_shared<sorted_value>();
sv.val = std::move(value_string); sv->val = std::move(value_string);
sv.orig = v; sv->orig = v;
sorted_values.push_back(std::move(sv)); sorted_values.push_back(std::move(sv));
} }
std::stable_sort(sorted_values.begin(), sorted_values.end()); std::stable_sort(sorted_values.begin(), sorted_values.end(), sorted_value());
std::vector<size_t> mapping; std::vector<size_t> mapping;
mapping.resize(sorted_values.size()); mapping.resize(sorted_values.size());
size_t value_index = 0; size_t value_index = 0;
for (size_t v = 0; v < sorted_values.size(); v++) { for (size_t v = 0; v < sorted_values.size(); v++) {
mapping[sorted_values[v].orig] = value_index; mapping[sorted_values[v]->orig] = value_index;
layer_writer.add_message(4, sorted_values[v].val); layer_writer.add_message(4, sorted_values[v]->val);
// crunch out duplicates that were missed by the hashing // crunch out duplicates that were missed by the hashing
while (v + 1 < sorted_values.size() && sorted_values[v].val == sorted_values[v + 1].val) { while (v + 1 < sorted_values.size() && sorted_values[v]->val == sorted_values[v + 1]->val) {
mapping[sorted_values[v + 1].orig] = value_index; sorted_values[v]->val.clear();
mapping[sorted_values[v + 1]->orig] = value_index;
v++; v++;
} }
sorted_values[v]->val.clear();
value_index++; value_index++;
} }
sorted_values.clear();
for (size_t f = 0; f < layers[i].features.size(); f++) { for (size_t f = 0; f < layers[i].features.size(); f++) {
std::string feature_string; std::string feature_string;
protozero::pbf_writer feature_writer(feature_string); protozero::pbf_writer feature_writer(feature_string);
tile.cpp
+3
View File
@@ -2574,6 +2574,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
layer.tag(feature, *layer_features[x]->full_keys[a], v); layer.tag(feature, *layer_features[x]->full_keys[a], v);
} }
layer_features[x]->full_keys.clear();
layer_features[x]->full_values.clear();
if (additional[A_CALCULATE_FEATURE_DENSITY]) { if (additional[A_CALCULATE_FEATURE_DENSITY]) {
int glow = 255; int glow = 255;
if (layer_features[x]->spacing > 0) { if (layer_features[x]->spacing > 0) {