diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Speeding up tippecanoe-overzoom (#191)

Browse Source 
* Speed up mvt_value comparison

* Converting repetitive ifs to cases

* More conversions from ifs to cases

* Optimize the always-true filter case

* Don't convert types of attributes without accumulators

* Unordered map seems to be faster than map

* Add missing header

* Fix some warnings

* Fix the warnings better

* Avoid an int->string->int conversion

* Lazily initialize layer key and values maps when actually needed

* More switches from maps to unordered_maps

* Sure, I'll take the microoptimization

* More emplacement

* Save some copies

* Emplaces and moves

* Lazy linear scan of attributes instead of building a map

* Extra printfs, missing header

* Avoid clipping if the input and output tiles are the same

* But do clip if the tile extent is being reduced

* Make sure I'm not constructing std::strings here at runtime

* More worrying about runtime string construction

* A couple more std::moves

* Const references!

* More const references

* Another std::move

* Make the string_value of mvt_value std::optional

* Reserve storage when decoding

* Provision for different mvt_values to share a string pool

* Use the string pool when decoding

* Avoid another string construction

* Try limiting the depth of the search for duplicate attributes

* Revert "Try limiting the depth of the search for duplicate attributes"

This reverts commit 9ec94a15ff.

* Update changelog

* Fix typo noticed during code review
This commit is contained in:
Erica Fischer
2024-01-29 11:33:56 -08:00
committed by GitHub
parent cbf222754b
commit 7d4d264d50
20 changed files with 553 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mvt.cpp
+183 -66
View File
@@ -110,7 +110,7 @@ int compress(std::string const &input, std::string &output, bool gz) {
return 0;
}
bool mvt_tile::decode(std::string &message, bool &was_compressed) {
bool mvt_tile::decode(const std::string &message, bool &was_compressed) {
layers.clear();
std::string src;
@@ -127,6 +127,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
}
protozero::pbf_reader reader(src);
std::shared_ptr<std::string> string_pool = std::make_shared<std::string>();
while (reader.next()) {
switch (reader.tag()) {
@@ -157,7 +158,12 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
switch (value_reader.tag()) {
case 1: /* string */
value.type = mvt_string;
value.string_value = value_reader.get_string();
value.s = string_pool;
{
auto v = value_reader.get_view();
std::string_view sv(v.data(), v.size());
value.set_string_value(sv);
}
break;
case 2: /* float */
@@ -196,7 +202,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
}
}
layer.values.push_back(value);
layer.values.push_back(std::move(value));
break;
}
@@ -224,6 +230,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
case 2: /* tag */
{
auto pi = feature_reader.get_packed_uint32();
feature.tags.reserve(std::distance(pi.first, pi.second));
for (auto it = pi.first; it != pi.second; ++it) {
feature.tags.push_back(*it);
}
@@ -237,6 +244,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
case 4: /* geometry */
{
auto pi = feature_reader.get_packed_uint32();
geoms.reserve(std::distance(pi.first, pi.second));
for (auto it = pi.first; it != pi.second; ++it) {
geoms.push_back(*it);
}
@@ -250,6 +258,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
}
long long px = 0, py = 0;
feature.geometry.reserve(geoms.size()); // probably not quite right, but still plausible
for (size_t g = 0; g < geoms.size(); g++) {
uint32_t geom = geoms[g];
uint32_t op = geom & 7;
@@ -261,14 +270,14 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
py += protozero::decode_zigzag32(geoms[g + 2]);
g += 2;
feature.geometry.push_back(mvt_geometry(op, px, py));
feature.geometry.emplace_back(op, px, py);
}
} else {
feature.geometry.push_back(mvt_geometry(op, 0, 0));
feature.geometry.emplace_back(op, 0, 0);
}
}
layer.features.push_back(feature);
layer.features.push_back(std::move(feature));
break;
}
@@ -278,14 +287,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
}
}
for (size_t i = 0; i < layer.keys.size(); i++) {
layer.key_map.insert(std::pair<std::string, size_t>(layer.keys[i], i));
}
for (size_t i = 0; i < layer.values.size(); i++) {
layer.value_map.insert(std::pair<mvt_value, size_t>(layer.values[i], i));
}
layers.push_back(layer);
layers.push_back(std::move(layer));
break;
}
@@ -313,7 +315,6 @@ struct sorted_value {
std::string mvt_tile::encode() {
std::string data;
protozero::pbf_writer writer(data);
for (size_t i = 0; i < layers.size(); i++) {
@@ -335,32 +336,40 @@ std::string mvt_tile::encode() {
protozero::pbf_writer value_writer(value_string);
mvt_value &pbv = layers[i].values[v];
if (pbv.type == mvt_string) {
value_writer.add_string(1, pbv.string_value);
} else if (pbv.type == mvt_float) {
switch (pbv.type) {
case mvt_string:
value_writer.add_string(1, pbv.get_string_value());
break;
case mvt_float:
value_writer.add_float(2, pbv.numeric_value.float_value);
} else if (pbv.type == mvt_double) {
break;
case mvt_double:
value_writer.add_double(3, pbv.numeric_value.double_value);
} else if (pbv.type == mvt_int) {
break;
case mvt_int:
value_writer.add_int64(4, pbv.numeric_value.int_value);
} else if (pbv.type == mvt_uint) {
break;
case mvt_uint:
value_writer.add_uint64(5, pbv.numeric_value.uint_value);
} else if (pbv.type == mvt_sint) {
break;
case mvt_sint:
value_writer.add_sint64(6, pbv.numeric_value.sint_value);
} else if (pbv.type == mvt_bool) {
break;
case mvt_bool:
value_writer.add_bool(7, pbv.numeric_value.bool_value);
} else if (pbv.type == mvt_null) {
break;
case mvt_null:
fprintf(stderr, "Internal error: trying to write null attribute to tile\n");
exit(EXIT_IMPOSSIBLE);
} else {
default:
fprintf(stderr, "Internal error: trying to write undefined attribute type to tile\n");
exit(EXIT_IMPOSSIBLE);
}
sorted_value sv;
sv.val = value_string;
sv.val = std::move(value_string);
sv.orig = v;
sorted_values.push_back(sv);
sorted_values.push_back(std::move(sv));
}
std::sort(sorted_values.begin(), sorted_values.end());
@@ -456,15 +465,70 @@ bool mvt_value::operator<(const mvt_value &o) const {
return true;
}
if (type == o.type) {
if ((type == mvt_string && string_value < o.string_value) ||
(type == mvt_float && numeric_value.float_value < o.numeric_value.float_value) ||
(type == mvt_double && numeric_value.double_value < o.numeric_value.double_value) ||
(type == mvt_int && numeric_value.int_value < o.numeric_value.int_value) ||
(type == mvt_uint && numeric_value.uint_value < o.numeric_value.uint_value) ||
(type == mvt_sint && numeric_value.sint_value < o.numeric_value.sint_value) ||
(type == mvt_bool && numeric_value.bool_value < o.numeric_value.bool_value) ||
(type == mvt_null && numeric_value.null_value < o.numeric_value.null_value)) {
return true;
switch (type) {
case mvt_string:
return get_string_view() < o.get_string_view();
case mvt_float:
return numeric_value.float_value < o.numeric_value.float_value;
case mvt_double:
return numeric_value.double_value < o.numeric_value.double_value;
case mvt_int:
return numeric_value.int_value < o.numeric_value.int_value;
case mvt_uint:
return numeric_value.uint_value < o.numeric_value.uint_value;
case mvt_sint:
return numeric_value.sint_value < o.numeric_value.sint_value;
case mvt_bool:
return numeric_value.bool_value < o.numeric_value.bool_value;
case mvt_null:
return numeric_value.null_value < o.numeric_value.null_value;
default:
fprintf(stderr, "mvt_value::operator<<: can't happen\n");
exit(EXIT_IMPOSSIBLE);
}
}
return false;
}
bool mvt_value::operator==(const mvt_value &o) const {
if (type == o.type) {
switch (type) {
case mvt_string:
return get_string_view() == o.get_string_view();
case mvt_float:
return numeric_value.float_value == o.numeric_value.float_value;
case mvt_double:
return numeric_value.double_value == o.numeric_value.double_value;
case mvt_int:
return numeric_value.int_value == o.numeric_value.int_value;
case mvt_uint:
return numeric_value.uint_value == o.numeric_value.uint_value;
case mvt_sint:
return numeric_value.sint_value == o.numeric_value.sint_value;
case mvt_bool:
return numeric_value.bool_value == o.numeric_value.bool_value;
case mvt_null:
return numeric_value.null_value == o.numeric_value.null_value;
default:
fprintf(stderr, "mvt_value::operator==: can't happen\n");
exit(EXIT_IMPOSSIBLE);
}
}
@@ -493,47 +557,62 @@ static std::string quote(std::string const &s) {
}
std::string mvt_value::toString() const {
if (type == mvt_string) {
return quote(string_value);
} else if (type == mvt_int) {
switch (type) {
case mvt_string:
return quote(get_string_value());
case mvt_int:
return std::to_string(numeric_value.int_value);
} else if (type == mvt_double) {
case mvt_double: {
double v = numeric_value.double_value;
if (v == (long long) v) {
return std::to_string((long long) v);
} else {
return milo::dtoa_milo(v);
}
} else if (type == mvt_float) {
}
case mvt_float: {
double v = numeric_value.float_value;
if (v == (long long) v) {
return std::to_string((long long) v);
} else {
return milo::dtoa_milo(v);
}
} else if (type == mvt_sint) {
}
case mvt_sint:
return std::to_string(numeric_value.sint_value);
} else if (type == mvt_uint) {
case mvt_uint:
return std::to_string(numeric_value.uint_value);
} else if (type == mvt_bool) {
case mvt_bool:
return numeric_value.bool_value ? "true" : "false";
} else if (type == mvt_null) {
case mvt_null:
return "null";
} else {
default:
return "unknown";
}
}
void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) {
void mvt_layer::tag(mvt_feature &feature, std::string const &key, mvt_value const &value) {
size_t ko, vo;
std::map<std::string, size_t>::iterator ki = key_map.find(key);
std::map<mvt_value, size_t>::iterator vi = value_map.find(value);
// initialize lazily the first time anyone tags an attribute
// to save the time of doing it in decode, which never actually matters.
// only tile writers actually need this.
if (key_map.size() == 0) {
for (size_t i = 0; i < keys.size(); i++) {
key_map.emplace(keys[i], i);
}
for (size_t i = 0; i < values.size(); i++) {
value_map.emplace(values[i], i);
}
}
std::unordered_map<std::string, size_t>::iterator ki = key_map.find(key);
std::unordered_map<mvt_value, size_t>::iterator vi = value_map.find(value);
if (ki == key_map.end()) {
ko = keys.size();
keys.push_back(key);
key_map.insert(std::pair<std::string, size_t>(key, ko));
key_map.emplace(key, ko);
} else {
ko = ki->second;
}
@@ -541,7 +620,7 @@ void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) {
if (vi == value_map.end()) {
vo = values.size();
values.push_back(value);
value_map.insert(std::pair<mvt_value, size_t>(value, vo));
value_map.emplace(value, vo);
} else {
vo = vi->second;
}
@@ -629,7 +708,8 @@ bool is_unsigned_integer(const char *s, unsigned long long *v) {
mvt_value stringified_to_mvt_value(int type, const char *s) {
mvt_value tv;
if (type == mvt_double) {
switch (type) {
case mvt_double: {
long long v;
unsigned long long uv;
if (is_unsigned_integer(s, &uv)) {
@@ -668,15 +748,18 @@ mvt_value stringified_to_mvt_value(int type, const char *s) {
}
}
}
} else if (type == mvt_bool) {
} break;
case mvt_bool:
tv.type = mvt_bool;
tv.numeric_value.bool_value = (s[0] == 't');
} else if (type == mvt_null) {
break;
case mvt_null:
tv.type = mvt_null;
tv.numeric_value.null_value = 0;
} else {
break;
default:
tv.type = mvt_string;
tv.string_value = s;
tv.set_string_value(s);
}
return tv;
@@ -690,34 +773,68 @@ mvt_value stringified_to_mvt_value(int type, const char *s) {
serial_val mvt_value_to_serial_val(mvt_value const &v) {
serial_val sv;
if (v.type == mvt_string) {
switch (v.type) {
case mvt_string:
sv.type = mvt_string;
sv.s = v.string_value;
} else if (v.type == mvt_float) {
sv.s = v.get_string_value();
break;
case mvt_float:
sv.type = mvt_double;
sv.s = milo::dtoa_milo(v.numeric_value.float_value);
} else if (v.type == mvt_double) {
break;
case mvt_double:
sv.type = mvt_double;
sv.s = milo::dtoa_milo(v.numeric_value.double_value);
} else if (v.type == mvt_int) {
break;
case mvt_int:
sv.type = mvt_double;
sv.s = std::to_string(v.numeric_value.int_value);
} else if (v.type == mvt_uint) {
break;
case mvt_uint:
sv.type = mvt_double;
sv.s = std::to_string(v.numeric_value.uint_value);
} else if (v.type == mvt_sint) {
break;
case mvt_sint:
sv.type = mvt_double;
sv.s = std::to_string(v.numeric_value.sint_value);
} else if (v.type == mvt_bool) {
break;
case mvt_bool:
sv.type = mvt_bool;
sv.s = v.numeric_value.bool_value ? "true" : "false";
} else if (v.type == mvt_null) {
break;
case mvt_null:
sv.type = mvt_null;
sv.s = "null";
} else {
break;
default:
fprintf(stderr, "unhandled mvt_type %d\n", v.type);
exit(EXIT_IMPOSSIBLE);
}
return sv;
}
// This extracts an integer value from an mvt_value
long long mvt_value_to_long_long(mvt_value const &v) {
switch (v.type) {
case mvt_string:
return atoll(v.c_str());
case mvt_float:
return v.numeric_value.float_value;
case mvt_double:
return v.numeric_value.double_value;
case mvt_int:
return v.numeric_value.int_value;
case mvt_uint:
return v.numeric_value.uint_value;
case mvt_sint:
return v.numeric_value.sint_value;
case mvt_bool:
return v.numeric_value.bool_value;
case mvt_null:
return 0;
default:
fprintf(stderr, "unhandled mvt_type %d\n", v.type);
exit(EXIT_IMPOSSIBLE);
}
}