mirror of
https://github.com/felt/tippecanoe.git
synced 2026-10-02 08:25:40 +02:00
Speeding up tippecanoe-overzoom (#191)
* Speed up mvt_value comparison
* Converting repetitive ifs to cases
* More conversions from ifs to cases
* Optimize the always-true filter case
* Don't convert types of attributes without accumulators
* Unordered map seems to be faster than map
* Add missing header
* Fix some warnings
* Fix the warnings better
* Avoid an int->string->int conversion
* Lazily initialize layer key and values maps when actually needed
* More switches from maps to unordered_maps
* Sure, I'll take the microoptimization
* More emplacement
* Save some copies
* Emplaces and moves
* Lazy linear scan of attributes instead of building a map
* Extra printfs, missing header
* Avoid clipping if the input and output tiles are the same
* But do clip if the tile extent is being reduced
* Make sure I'm not constructing std::strings here at runtime
* More worrying about runtime string construction
* A couple more std::moves
* Const references!
* More const references
* Another std::move
* Make the string_value of mvt_value std::optional
* Reserve storage when decoding
* Provision for different mvt_values to share a string pool
* Use the string pool when decoding
* Avoid another string construction
* Try limiting the depth of the search for duplicate attributes
* Revert "Try limiting the depth of the search for duplicate attributes"
This reverts commit 9ec94a15ff.
* Update changelog
* Fix typo noticed during code review
This commit is contained in:
Erica Fischer
+159
-103
@@ -1,37 +1,40 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
#include <functional>
|
||||
#include "mvt.hpp"
|
||||
#include "evaluator.hpp"
|
||||
#include "errors.hpp"
|
||||
#include "milo/dtoa_milo.h"
|
||||
|
||||
static std::string mvt_value_to_string(mvt_value one, bool &fail) {
|
||||
if (one.type == mvt_string) {
|
||||
return one.string_value;
|
||||
} else if (one.type == mvt_float) {
|
||||
static std::string mvt_value_to_string(mvt_value const &one, bool &fail) {
|
||||
switch (one.type) {
|
||||
case mvt_string:
|
||||
return one.get_string_value();
|
||||
case mvt_float:
|
||||
return milo::dtoa_milo(one.numeric_value.float_value);
|
||||
} else if (one.type == mvt_double) {
|
||||
case mvt_double:
|
||||
return milo::dtoa_milo(one.numeric_value.double_value);
|
||||
} else if (one.type == mvt_int) {
|
||||
case mvt_int:
|
||||
return std::to_string(one.numeric_value.int_value);
|
||||
} else if (one.type == mvt_uint) {
|
||||
case mvt_uint:
|
||||
return std::to_string(one.numeric_value.uint_value);
|
||||
} else if (one.type == mvt_sint) {
|
||||
case mvt_sint:
|
||||
return std::to_string(one.numeric_value.sint_value);
|
||||
} else if (one.type == mvt_bool) {
|
||||
case mvt_bool:
|
||||
return one.numeric_value.bool_value ? "true" : "false";
|
||||
} else if (one.type == mvt_null) {
|
||||
case mvt_null:
|
||||
fail = true; // null op string => null
|
||||
return "";
|
||||
} else {
|
||||
case mvt_no_such_key:
|
||||
default:
|
||||
fprintf(stderr, "unhandled mvt_type %d\n", one.type);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
int compare_fsl(mvt_value one, json_object *two, bool &fail) {
|
||||
int compare_fsl(mvt_value const &one, json_object *two, bool &fail) {
|
||||
// In FSL expressions, the attribute value is coerced to the type
|
||||
// of the JSON literal value it is being compared to.
|
||||
//
|
||||
@@ -47,30 +50,38 @@ int compare_fsl(mvt_value one, json_object *two, bool &fail) {
|
||||
if (two->type == JSON_NUMBER) {
|
||||
double lhs;
|
||||
|
||||
if (one.type == mvt_string) {
|
||||
switch (one.type) {
|
||||
case mvt_string: {
|
||||
char *endptr = NULL;
|
||||
const char *s = one.string_value.c_str();
|
||||
const char *s = one.c_str();
|
||||
lhs = strtod(s, &endptr);
|
||||
if (endptr == s) {
|
||||
fail = true; // non-numeric-string op number => null
|
||||
return 0;
|
||||
}
|
||||
} else if (one.type == mvt_float) {
|
||||
} break;
|
||||
case mvt_float:
|
||||
lhs = one.numeric_value.float_value;
|
||||
} else if (one.type == mvt_double) {
|
||||
break;
|
||||
case mvt_double:
|
||||
lhs = one.numeric_value.double_value;
|
||||
} else if (one.type == mvt_int) {
|
||||
break;
|
||||
case mvt_int:
|
||||
lhs = one.numeric_value.int_value;
|
||||
} else if (one.type == mvt_uint) {
|
||||
break;
|
||||
case mvt_uint:
|
||||
lhs = one.numeric_value.uint_value;
|
||||
} else if (one.type == mvt_sint) {
|
||||
break;
|
||||
case mvt_sint:
|
||||
lhs = one.numeric_value.sint_value;
|
||||
} else if (one.type == mvt_bool) {
|
||||
break;
|
||||
case mvt_bool:
|
||||
lhs = one.numeric_value.bool_value;
|
||||
} else if (one.type == mvt_null) {
|
||||
break;
|
||||
case mvt_null:
|
||||
fail = true; // null op number => null
|
||||
return 0;
|
||||
} else {
|
||||
default:
|
||||
fprintf(stderr, "unhandled mvt_type %d\n", one.type);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
@@ -93,24 +104,34 @@ int compare_fsl(mvt_value one, json_object *two, bool &fail) {
|
||||
if (two->type == JSON_TRUE || two->type == JSON_FALSE) {
|
||||
bool lhs;
|
||||
|
||||
if (one.type == mvt_string) {
|
||||
lhs = one.string_value.size() > 0;
|
||||
} else if (one.type == mvt_float) {
|
||||
switch (one.type) {
|
||||
case mvt_string:
|
||||
lhs = one.get_string_view().size() > 0;
|
||||
break;
|
||||
case mvt_float:
|
||||
lhs = one.numeric_value.float_value != 0;
|
||||
} else if (one.type == mvt_double) {
|
||||
break;
|
||||
case mvt_double:
|
||||
lhs = one.numeric_value.double_value != 0;
|
||||
} else if (one.type == mvt_int) {
|
||||
break;
|
||||
case mvt_int:
|
||||
lhs = one.numeric_value.int_value != 0;
|
||||
} else if (one.type == mvt_uint) {
|
||||
break;
|
||||
case mvt_uint:
|
||||
lhs = one.numeric_value.uint_value != 0;
|
||||
} else if (one.type == mvt_sint) {
|
||||
break;
|
||||
case mvt_sint:
|
||||
lhs = one.numeric_value.sint_value != 0;
|
||||
} else if (one.type == mvt_bool) {
|
||||
break;
|
||||
case mvt_bool:
|
||||
lhs = one.numeric_value.bool_value;
|
||||
} else if (one.type == mvt_null) {
|
||||
break;
|
||||
case mvt_null:
|
||||
fail = true; // null op bool => null
|
||||
return 0;
|
||||
} else {
|
||||
|
||||
case mvt_no_such_key:
|
||||
default:
|
||||
fprintf(stderr, "unhandled mvt_type %d\n", one.type);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
@@ -123,34 +144,45 @@ int compare_fsl(mvt_value one, json_object *two, bool &fail) {
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
int compare(mvt_value one, json_object *two, bool &fail) {
|
||||
if (one.type == mvt_string) {
|
||||
int compare(mvt_value const &one, json_object *two, bool &fail) {
|
||||
switch (one.type) {
|
||||
case mvt_string:
|
||||
if (two->type != JSON_STRING) {
|
||||
fail = true;
|
||||
return false; // string vs non-string
|
||||
}
|
||||
|
||||
return strcmp(one.string_value.c_str(), two->value.string.string);
|
||||
}
|
||||
return strcmp(one.c_str(), two->value.string.string);
|
||||
|
||||
if (one.type == mvt_double || one.type == mvt_float || one.type == mvt_int || one.type == mvt_uint || one.type == mvt_sint) {
|
||||
case mvt_double:
|
||||
case mvt_float:
|
||||
case mvt_int:
|
||||
case mvt_uint:
|
||||
case mvt_sint:
|
||||
if (two->type != JSON_NUMBER) {
|
||||
fail = true;
|
||||
return false; // number vs non-number
|
||||
}
|
||||
|
||||
double v;
|
||||
if (one.type == mvt_double) {
|
||||
switch (one.type) {
|
||||
case mvt_double:
|
||||
v = one.numeric_value.double_value;
|
||||
} else if (one.type == mvt_float) {
|
||||
break;
|
||||
case mvt_float:
|
||||
v = one.numeric_value.float_value;
|
||||
} else if (one.type == mvt_int) {
|
||||
break;
|
||||
case mvt_int:
|
||||
v = one.numeric_value.int_value;
|
||||
} else if (one.type == mvt_uint) {
|
||||
break;
|
||||
case mvt_uint:
|
||||
v = one.numeric_value.uint_value;
|
||||
} else if (one.type == mvt_sint) {
|
||||
break;
|
||||
case mvt_sint:
|
||||
v = one.numeric_value.sint_value;
|
||||
} else {
|
||||
break;
|
||||
case mvt_no_such_key:
|
||||
default:
|
||||
fprintf(stderr, "Internal error: bad mvt type %d\n", one.type);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
@@ -162,25 +194,29 @@ int compare(mvt_value one, json_object *two, bool &fail) {
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (one.type == mvt_bool) {
|
||||
case mvt_bool:
|
||||
if (two->type != JSON_TRUE && two->type != JSON_FALSE) {
|
||||
fail = true;
|
||||
return false; // bool vs non-bool
|
||||
}
|
||||
|
||||
bool b = two->type != JSON_FALSE;
|
||||
return one.numeric_value.bool_value > b;
|
||||
}
|
||||
{
|
||||
bool b = two->type != JSON_FALSE;
|
||||
return one.numeric_value.bool_value > b;
|
||||
}
|
||||
|
||||
if (one.type == mvt_null) {
|
||||
case mvt_null:
|
||||
if (two->type != JSON_NULL) {
|
||||
fail = true;
|
||||
return false; // null vs non-null
|
||||
}
|
||||
|
||||
return 0; // null equals null
|
||||
|
||||
case mvt_no_such_key:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
fprintf(stderr, "Internal error: bad mvt type %d\n", one.type);
|
||||
@@ -190,7 +226,7 @@ int compare(mvt_value one, json_object *two, bool &fail) {
|
||||
// 0: false
|
||||
// 1: true
|
||||
// -1: incomparable (sql null), treated as false in final output
|
||||
static int eval(std::map<std::string, mvt_value> const &feature, json_object *f, std::set<std::string> &exclude_attributes) {
|
||||
static int eval(std::function<mvt_value(std::string const &)> feature, json_object *f, std::set<std::string> &exclude_attributes) {
|
||||
if (f != NULL) {
|
||||
if (f->type == JSON_TRUE) {
|
||||
return 1;
|
||||
@@ -236,10 +272,10 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
|
||||
if (f->value.array.array[0]->type == JSON_STRING) {
|
||||
// if LHS of a boolean conjunction is a string, it is an attribute reference
|
||||
auto ff = feature.find(std::string(f->value.array.array[0]->value.string.string));
|
||||
if (ff != feature.end()) {
|
||||
if (ff->second.type == mvt_bool) {
|
||||
lhs = ff->second.numeric_value.bool_value;
|
||||
mvt_value ff = feature(std::string(f->value.array.array[0]->value.string.string));
|
||||
if (ff.type != mvt_no_such_key) {
|
||||
if (ff.type == mvt_bool) {
|
||||
lhs = ff.numeric_value.bool_value;
|
||||
} else {
|
||||
lhs = -1; // not boolean: null
|
||||
}
|
||||
@@ -280,9 +316,9 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
false)) {
|
||||
mvt_value lhs;
|
||||
lhs.type = mvt_null; // attributes that aren't found are nulls
|
||||
auto ff = feature.find(std::string(f->value.array.array[0]->value.string.string));
|
||||
if (ff != feature.end()) {
|
||||
lhs = ff->second;
|
||||
mvt_value ff = feature(std::string(f->value.array.array[0]->value.string.string));
|
||||
if (ff.type != mvt_no_such_key) {
|
||||
lhs = ff;
|
||||
}
|
||||
|
||||
if (f->value.array.array[2]->type == JSON_NULL && strcmp(f->value.array.array[1]->value.string.string, "is") == 0) {
|
||||
@@ -341,9 +377,8 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
}
|
||||
}
|
||||
|
||||
int cmp = compare_fsl(ff->second, f->value.array.array[2], fail);
|
||||
int cmp = compare_fsl(ff, f->value.array.array[2], fail);
|
||||
if (fail) {
|
||||
printf("cast fail\n");
|
||||
return -1; // null
|
||||
}
|
||||
|
||||
@@ -387,7 +422,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f));
|
||||
exit(EXIT_FILTER);
|
||||
}
|
||||
return feature.count(std::string(f->value.array.array[1]->value.string.string)) != 0;
|
||||
return feature(std::string(f->value.array.array[1]->value.string.string)).type != mvt_no_such_key;
|
||||
}
|
||||
|
||||
if (strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) {
|
||||
@@ -395,7 +430,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f));
|
||||
exit(EXIT_FILTER);
|
||||
}
|
||||
return feature.count(std::string(f->value.array.array[1]->value.string.string)) == 0;
|
||||
return feature(std::string(f->value.array.array[1]->value.string.string)).type == mvt_no_such_key;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,8 +449,8 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
exit(EXIT_FILTER);
|
||||
}
|
||||
|
||||
auto ff = feature.find(std::string(f->value.array.array[1]->value.string.string));
|
||||
if (ff == feature.end()) {
|
||||
mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string));
|
||||
if (ff.type == mvt_no_such_key) {
|
||||
static bool warned = false;
|
||||
if (!warned) {
|
||||
const char *s = json_stringify(f);
|
||||
@@ -430,7 +465,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
}
|
||||
|
||||
bool fail = false;
|
||||
int cmp = compare(ff->second, f->value.array.array[2], fail);
|
||||
int cmp = compare(ff, f->value.array.array[2], fail);
|
||||
|
||||
if (fail) {
|
||||
static bool warned = false;
|
||||
@@ -517,8 +552,8 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
exit(EXIT_FILTER);
|
||||
}
|
||||
|
||||
auto ff = feature.find(std::string(f->value.array.array[1]->value.string.string));
|
||||
if (ff == feature.end()) {
|
||||
mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string));
|
||||
if (ff.type == mvt_no_such_key) {
|
||||
static bool warned = false;
|
||||
if (!warned) {
|
||||
const char *s = json_stringify(f);
|
||||
@@ -535,7 +570,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
bool found = false;
|
||||
for (size_t i = 2; i < f->value.array.length; i++) {
|
||||
bool fail = false;
|
||||
int cmp = compare(ff->second, f->value.array.array[i], fail);
|
||||
int cmp = compare(ff, f->value.array.array[i], fail);
|
||||
|
||||
if (fail) {
|
||||
static bool warned = false;
|
||||
@@ -584,7 +619,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
|
||||
exit(EXIT_FILTER);
|
||||
}
|
||||
|
||||
bool evaluate(std::map<std::string, mvt_value> const &feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes) {
|
||||
bool evaluate(std::function<mvt_value(std::string const &)> feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes) {
|
||||
if (filter == NULL || filter->type != JSON_HASH) {
|
||||
fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter));
|
||||
exit(EXIT_JSON);
|
||||
@@ -638,48 +673,69 @@ json_object *parse_filter(const char *s) {
|
||||
return filter;
|
||||
}
|
||||
|
||||
bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set<std::string> &exclude_attributes, int z) {
|
||||
if (filter != NULL) {
|
||||
std::map<std::string, mvt_value> attributes;
|
||||
|
||||
for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) {
|
||||
std::string key = layer.keys[feat.tags[t]];
|
||||
const mvt_value &val = layer.values[feat.tags[t + 1]];
|
||||
|
||||
attributes.insert(std::pair<std::string, mvt_value>(key, val));
|
||||
bool evaluate(std::unordered_map<std::string, mvt_value> const &feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes) {
|
||||
std::function<mvt_value(std::string const &)> getter = [&](std::string const &key) {
|
||||
auto f = feature.find(key);
|
||||
if (f != feature.end()) {
|
||||
return f->second;
|
||||
} else {
|
||||
mvt_value v;
|
||||
v.type = mvt_no_such_key;
|
||||
v.numeric_value.null_value = 0;
|
||||
return v;
|
||||
}
|
||||
};
|
||||
|
||||
if (feat.has_id) {
|
||||
return evaluate(getter, layer, filter, exclude_attributes);
|
||||
}
|
||||
|
||||
bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set<std::string> &exclude_attributes, int z) {
|
||||
std::function<mvt_value(std::string const &)> getter = [&](std::string const &key) {
|
||||
const static std::string dollar_id = "$id";
|
||||
if (key == dollar_id && feat.has_id) {
|
||||
mvt_value v;
|
||||
v.type = mvt_uint;
|
||||
v.numeric_value.uint_value = feat.id;
|
||||
return v;
|
||||
}
|
||||
|
||||
attributes.insert(std::pair<std::string, mvt_value>("$id", v));
|
||||
const static std::string dollar_type = "$type";
|
||||
if (key == dollar_type) {
|
||||
mvt_value v;
|
||||
v.type = mvt_string;
|
||||
|
||||
if (feat.type == mvt_point) {
|
||||
const static std::string point = "Point";
|
||||
v.set_string_value(point);
|
||||
} else if (feat.type == mvt_linestring) {
|
||||
const static std::string linestring = "LineString";
|
||||
v.set_string_value(linestring);
|
||||
} else if (feat.type == mvt_polygon) {
|
||||
const static std::string polygon = "Polygon";
|
||||
v.set_string_value(polygon);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
const static std::string dollar_zoom = "$zoom";
|
||||
if (key == dollar_zoom) {
|
||||
mvt_value v2;
|
||||
v2.type = mvt_uint;
|
||||
v2.numeric_value.uint_value = z;
|
||||
return v2;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i + 1 < feat.tags.size(); i += 2) {
|
||||
if (layer.keys[feat.tags[i]] == key) {
|
||||
return layer.values[feat.tags[i + 1]];
|
||||
}
|
||||
}
|
||||
|
||||
mvt_value v;
|
||||
v.type = mvt_string;
|
||||
v.type = mvt_no_such_key;
|
||||
v.numeric_value.null_value = 0;
|
||||
return v;
|
||||
};
|
||||
|
||||
if (feat.type == mvt_point) {
|
||||
v.string_value = "Point";
|
||||
} else if (feat.type == mvt_linestring) {
|
||||
v.string_value = "LineString";
|
||||
} else if (feat.type == mvt_polygon) {
|
||||
v.string_value = "Polygon";
|
||||
}
|
||||
|
||||
attributes.insert(std::pair<std::string, mvt_value>("$type", v));
|
||||
|
||||
mvt_value v2;
|
||||
v2.type = mvt_uint;
|
||||
v2.numeric_value.uint_value = z;
|
||||
|
||||
attributes.insert(std::pair<std::string, mvt_value>("$zoom", v2));
|
||||
|
||||
if (!evaluate(attributes, layer.name, filter, exclude_attributes)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
return evaluate(getter, layer.name, filter, exclude_attributes);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user