diff --git a/CHANGELOG.md b/CHANGELOG.md index 095a17fd..a8e19d96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 2.75.1 + +* Further reduce memory consumption in attribute sorting and tilestats tracking + # 2.75.0 * Reduce memory consumption in attribute accumulation and feature sorting diff --git a/mvt.cpp b/mvt.cpp index 8594fe51..16f9f583 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -316,6 +316,10 @@ struct sorted_value { return false; } + + bool operator()(const std::shared_ptr &a, const std::shared_ptr &b) { + return *a < *b; + } }; std::string mvt_tile::encode() { @@ -334,7 +338,7 @@ std::string mvt_tile::encode() { layer_writer.add_string(3, layers[i].keys[j]); /* key */ } - std::vector sorted_values; + std::vector> sorted_values; for (size_t v = 0; v < layers[i].values.size(); v++) { std::string value_string; @@ -371,30 +375,34 @@ std::string mvt_tile::encode() { exit(EXIT_IMPOSSIBLE); } - sorted_value sv; - sv.val = std::move(value_string); - sv.orig = v; + std::shared_ptr sv = std::make_shared(); + sv->val = std::move(value_string); + sv->orig = v; sorted_values.push_back(std::move(sv)); } - std::stable_sort(sorted_values.begin(), sorted_values.end()); + std::stable_sort(sorted_values.begin(), sorted_values.end(), sorted_value()); std::vector mapping; mapping.resize(sorted_values.size()); size_t value_index = 0; for (size_t v = 0; v < sorted_values.size(); v++) { - mapping[sorted_values[v].orig] = value_index; - layer_writer.add_message(4, sorted_values[v].val); + mapping[sorted_values[v]->orig] = value_index; + layer_writer.add_message(4, sorted_values[v]->val); // crunch out duplicates that were missed by the hashing - while (v + 1 < sorted_values.size() && sorted_values[v].val == sorted_values[v + 1].val) { - mapping[sorted_values[v + 1].orig] = value_index; + while (v + 1 < sorted_values.size() && sorted_values[v]->val == sorted_values[v + 1]->val) { + sorted_values[v]->val.clear(); + mapping[sorted_values[v + 1]->orig] = value_index; v++; } + sorted_values[v]->val.clear(); value_index++; } + sorted_values.clear(); + for (size_t f = 0; f < layers[i].features.size(); f++) { std::string feature_string; protozero::pbf_writer feature_writer(feature_string); diff --git a/serial.hpp b/serial.hpp index 1c86cc05..a351f21d 100644 --- a/serial.hpp +++ b/serial.hpp @@ -165,7 +165,6 @@ struct serial_feature { long long clustered; // does this feature need the clustered/point_count attributes? const char *stringpool; // string pool for keys/values lookup std::shared_ptr tile_stringpool; // string pool for mvt_value construction - std::set need_tilestats; int z; // tile being produced int tx; diff --git a/tile.cpp b/tile.cpp index 0703ff97..40ef416c 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1405,10 +1405,6 @@ void add_tilestats(std::string const &layername, int z, std::vector= 0) { p.full_keys.push_back(key_pool.pool(prefixed_key)); p.full_values.push_back(p.full_values[found_as]); + return; } @@ -2300,8 +2293,6 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch std::vector> &features = kv.second.features; if (retain_points_multiplier > 1) { - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "tippecanoe:retain_points_multiplier_first", serial_val(mvt_bool, "true")); - // mapping from input sequence to current sequence within this tile std::vector> feature_sequences; @@ -2322,8 +2313,6 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch features[j]->full_keys.push_back(key_pool.pool("tippecanoe:retain_points_multiplier_sequence")); features[j]->full_values.push_back(sv); - - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *features[j]->full_keys.back(), sv); } } @@ -2340,22 +2329,16 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch sv.s = "true"; p.full_values.push_back(sv); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "clustered", sv); - p.full_keys.push_back(key_pool.pool("point_count")); sv2.type = mvt_double; sv2.s = std::to_string(point_count); p.full_values.push_back(sv2); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "point_count", sv2); - p.full_keys.push_back(key_pool.pool("sqrt_point_count")); sv3.type = mvt_double; sv3.s = std::to_string(round(100 * sqrt(point_count)) / 100.0); p.full_values.push_back(sv3); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "sqrt_point_count", sv3); - p.full_keys.push_back(key_pool.pool("point_count_abbreviated")); sv4.type = mvt_string; if (point_count >= 10000) { @@ -2367,21 +2350,15 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } sv4.s = abbrev; p.full_values.push_back(sv4); - - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "point_count_abbreviated", sv4); } - if (p.need_tilestats.size() > 0) { - for (size_t j = 0; j < p.full_keys.size(); j++) { - if (p.need_tilestats.count(*p.full_keys[j]) > 0) { - // remove accumulation state - size_t found = p.full_values[j].s.find('\0'); - if (found != std::string::npos) { - p.full_values[j].s = p.full_values[j].s.substr(0, found); - } - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]); - } + for (size_t j = 0; j < p.full_keys.size(); j++) { + // remove accumulation state + size_t found = p.full_values[j].s.find('\0'); + if (found != std::string::npos) { + p.full_values[j].s = p.full_values[j].s.substr(0, found); } + add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]); } } @@ -2574,6 +2551,9 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch layer.tag(feature, *layer_features[x]->full_keys[a], v); } + layer_features[x]->full_keys.clear(); + layer_features[x]->full_values.clear(); + if (additional[A_CALCULATE_FEATURE_DENSITY]) { int glow = 255; if (layer_features[x]->spacing > 0) { diff --git a/version.hpp b/version.hpp index 4b0af2fb..769bef5d 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v2.75.0" +#define VERSION "v2.75.1" #endif