diff --git a/CHANGELOG.md b/CHANGELOG.md index b5575856..b7e9f406 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Performance optimizations to tile reading, writing, and overzooming + # 2.41.2 * Add --accumulate-attribute to tippecanoe-overzoom diff --git a/attribute.cpp b/attribute.cpp index 1b12c960..b01104f1 100644 --- a/attribute.cpp +++ b/attribute.cpp @@ -1,12 +1,12 @@ #include -#include +#include #include "attribute.hpp" #include "errors.hpp" #include "serial.hpp" #include "jsonpull/jsonpull.h" #include "milo/dtoa_milo.h" -void set_attribute_accum(std::map &attribute_accum, std::string name, std::string type) { +void set_attribute_accum(std::unordered_map &attribute_accum, std::string name, std::string type) { attribute_op t; if (type == "sum") { @@ -31,7 +31,7 @@ void set_attribute_accum(std::map &attribute_accum, s attribute_accum.insert(std::pair(name, t)); } -void set_attribute_accum(std::map &attribute_accum, const char *arg, char **argv) { +void set_attribute_accum(std::unordered_map &attribute_accum, const char *arg, char **argv) { if (*arg == '{') { json_pull *jp = json_begin_string(arg); json_object *o = json_read_tree(jp); @@ -79,7 +79,7 @@ void set_attribute_accum(std::map &attribute_accum, c set_attribute_accum(attribute_accum, name, type); } -void preserve_attribute(attribute_op op, std::string &key, serial_val &val, std::vector &full_keys, std::vector &full_values, std::map &attribute_accum_state) { +void preserve_attribute(attribute_op op, std::string &key, serial_val &val, std::vector &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state) { for (size_t i = 0; i < full_keys.size(); i++) { if (key == full_keys[i]) { switch (op) { diff --git a/attribute.hpp b/attribute.hpp index c4d999e9..9d8ca41c 100644 --- a/attribute.hpp +++ b/attribute.hpp @@ -2,7 +2,7 @@ #define ATTRIBUTE_HPP #include -#include +#include enum attribute_op { op_sum, @@ -21,8 +21,8 @@ struct accum_state { struct serial_val; -void set_attribute_accum(std::map &attribute_accum, std::string name, std::string type); -void set_attribute_accum(std::map &attribute_accum, const char *arg, char **argv); -void preserve_attribute(attribute_op op, std::string &key, serial_val &val, std::vector &full_keys, std::vector &full_values, std::map &attribute_accum_state); +void set_attribute_accum(std::unordered_map &attribute_accum, std::string name, std::string type); +void set_attribute_accum(std::unordered_map &attribute_accum, const char *arg, char **argv); +void preserve_attribute(attribute_op op, std::string &key, serial_val &val, std::vector &full_keys, std::vector &full_values, std::unordered_map &attribute_accum_state); #endif diff --git a/clip.cpp b/clip.cpp index 4a704be2..83a97d85 100644 --- a/clip.cpp +++ b/clip.cpp @@ -755,10 +755,10 @@ static std::vector> clip_poly1(std::vector const &keep, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object *filter, bool preserve_input_order, std::map const &attribute_accum) { + bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map const &attribute_accum) { mvt_tile tile; try { @@ -785,7 +785,7 @@ struct tile_feature { size_t seq = 0; }; -static void feature_out(std::vector const &features, mvt_layer &outlayer, std::set const &keep, std::map const &attribute_accum) { +static void feature_out(std::vector const &features, mvt_layer &outlayer, std::set const &keep, std::unordered_map const &attribute_accum) { // Add geometry to output feature mvt_feature outfeature; @@ -810,13 +810,22 @@ static void feature_out(std::vector const &features, mvt_layer &ou // attributes from the other features of the // multiplier cluster accumulated onto them - std::map attribute_accum_state; + std::unordered_map attribute_accum_state; std::vector full_keys; std::vector full_values; for (size_t i = 0; i + 1 < features[0].tags.size(); i += 2) { - full_keys.push_back(features[0].layer->keys[features[0].tags[i]]); - full_values.push_back(mvt_value_to_serial_val(features[0].layer->values[features[0].tags[i + 1]])); + auto f = attribute_accum.find(features[0].layer->keys[features[0].tags[i]]); + if (f != attribute_accum.end()) { + // this attribute has an accumulator, so convert it + full_keys.push_back(features[0].layer->keys[features[0].tags[i]]); + full_values.push_back(mvt_value_to_serial_val(features[0].layer->values[features[0].tags[i + 1]])); + } else { + // otherwise just tag it directly onto the output feature + if (keep.size() == 0 || keep.find(features[0].layer->keys[features[0].tags[i]]) != keep.end()) { + outlayer.tag(outfeature, features[0].layer->keys[features[0].tags[i]], features[0].layer->values[features[0].tags[i + 1]]); + } + } } // accumulate whatever attributes are specified to be accumulated @@ -851,7 +860,7 @@ static void feature_out(std::vector const &features, mvt_layer &ou } } - outlayer.features.push_back(outfeature); + outlayer.features.push_back(std::move(outfeature)); } } @@ -861,10 +870,10 @@ static struct preservecmp { } } preservecmp; -std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny, +std::string overzoom(const mvt_tile &tile, int oz, int ox, int oy, int nz, int nx, int ny, int detail, int buffer, std::set const &keep, bool do_compress, std::vector> *next_overzoomed_tiles, - bool demultiply, json_object *filter, bool preserve_input_order, std::map const &attribute_accum) { + bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map const &attribute_accum) { mvt_tile outtile; for (auto const &layer : tile.layers) { @@ -881,11 +890,14 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int std::vector pending_tile_features; + static const std::string retain_points_multiplier_first = "tippecanoe:retain_points_multiplier_first"; + static const std::string retain_points_multiplier_sequence = "tippecanoe:retain_points_multiplier_sequence"; + for (auto feature : layer.features) { bool flush_multiplier_cluster = false; if (demultiply) { for (ssize_t i = feature.tags.size() - 2; i >= 0; i -= 2) { - if (layer.keys[feature.tags[i]] == "tippecanoe:retain_points_multiplier_first") { + if (layer.keys[feature.tags[i]] == retain_points_multiplier_first) { mvt_value v = layer.values[feature.tags[i + 1]]; if (v.type == mvt_bool && v.numeric_value.bool_value) { flush_multiplier_cluster = true; @@ -893,9 +905,9 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int } } - if (layer.keys[feature.tags[i]] == "tippecanoe:retain_points_multiplier_sequence") { + if (layer.keys[feature.tags[i]] == retain_points_multiplier_sequence) { mvt_value v = layer.values[feature.tags[i + 1]]; - feature.seq = atoll(mvt_value_to_serial_val(v).s.c_str()); + feature.seq = mvt_value_to_long_long(v); feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2); } } @@ -911,7 +923,7 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int } std::set exclude_attributes; - if (!evaluate(feature, layer, filter, exclude_attributes, nz)) { + if (filter != NULL && !evaluate(feature, layer, filter, exclude_attributes, nz)) { continue; } @@ -922,6 +934,7 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int long long tilesize = 1LL << (32 - oz); // source tile size in world coordinates draw ring_closure(0, 0, 0); + bool sametile = (nz == oz && nx == ox && ny == oy && outlayer.extent >= layer.extent); for (auto const &g : feature.geometry) { if (g.op == mvt_closepath) { @@ -947,52 +960,59 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int g.y -= ny * outtilesize; } - // Clip to output tile + if (!sametile) { + // Clip to output tile - long long xmin = LLONG_MAX; - long long ymin = LLONG_MAX; - long long xmax = LLONG_MIN; - long long ymax = LLONG_MIN; + long long xmin = LLONG_MAX; + long long ymin = LLONG_MAX; + long long xmax = LLONG_MIN; + long long ymax = LLONG_MIN; - for (auto const &g : geom) { - xmin = std::min(xmin, g.x); - ymin = std::min(ymin, g.y); - xmax = std::max(xmax, g.x); - ymax = std::max(ymax, g.y); - } + for (auto const &g : geom) { + xmin = std::min(xmin, g.x); + ymin = std::min(ymin, g.y); + xmax = std::max(xmax, g.x); + ymax = std::max(ymax, g.y); + } - long long b = outtilesize * buffer / 256; - if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) { - continue; - } + long long b = outtilesize * buffer / 256; + if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) { + continue; + } - if (t == VT_LINE) { - geom = clip_lines(geom, nz, buffer); - } else if (t == VT_POLYGON) { - drawvec dv; - geom = simple_clip_poly(geom, nz, buffer, dv, false); - } else if (t == VT_POINT) { - geom = clip_point(geom, nz, buffer); + if (t == VT_LINE) { + geom = clip_lines(geom, nz, buffer); + } else if (t == VT_POLYGON) { + drawvec dv; + geom = simple_clip_poly(geom, nz, buffer, dv, false); + } else if (t == VT_POINT) { + geom = clip_point(geom, nz, buffer); + } } // Scale to output tile extent to_tile_scale(geom, nz, det); - // Clean geometries + if (!sametile) { + // Clean geometries + + geom = remove_noop(geom, t, 0); + if (t == VT_POLYGON) { + geom = clean_or_clip_poly(geom, 0, 0, false, false); + } + } - geom = remove_noop(geom, t, 0); if (t == VT_POLYGON) { - geom = clean_or_clip_poly(geom, 0, 0, false, false); geom = close_poly(geom); } tile_feature tf; - tf.geom = geom; + tf.geom = std::move(geom); tf.t = t; tf.has_id = feature.has_id; tf.id = feature.id; - tf.tags = feature.tags; + tf.tags = std::move(feature.tags); tf.layer = &layer; tf.seq = feature.seq; @@ -1009,7 +1029,7 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int } if (outlayer.features.size() > 0) { - outtile.layers.push_back(outlayer); + outtile.layers.push_back(std::move(outlayer)); } } diff --git a/decode.cpp b/decode.cpp index 4bd64849..dedb4053 100644 --- a/decode.cpp +++ b/decode.cpp @@ -277,7 +277,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co struct stat st; std::vector tiles; - char *pmtiles_map; + char *pmtiles_map = NULL; std::vector entries; bool is_pmtiles = false; diff --git a/evaluator.cpp b/evaluator.cpp index 9633bfce..7508f46c 100644 --- a/evaluator.cpp +++ b/evaluator.cpp @@ -1,37 +1,40 @@ #include #include #include -#include +#include +#include #include "mvt.hpp" #include "evaluator.hpp" #include "errors.hpp" #include "milo/dtoa_milo.h" -static std::string mvt_value_to_string(mvt_value one, bool &fail) { - if (one.type == mvt_string) { - return one.string_value; - } else if (one.type == mvt_float) { +static std::string mvt_value_to_string(mvt_value const &one, bool &fail) { + switch (one.type) { + case mvt_string: + return one.get_string_value(); + case mvt_float: return milo::dtoa_milo(one.numeric_value.float_value); - } else if (one.type == mvt_double) { + case mvt_double: return milo::dtoa_milo(one.numeric_value.double_value); - } else if (one.type == mvt_int) { + case mvt_int: return std::to_string(one.numeric_value.int_value); - } else if (one.type == mvt_uint) { + case mvt_uint: return std::to_string(one.numeric_value.uint_value); - } else if (one.type == mvt_sint) { + case mvt_sint: return std::to_string(one.numeric_value.sint_value); - } else if (one.type == mvt_bool) { + case mvt_bool: return one.numeric_value.bool_value ? "true" : "false"; - } else if (one.type == mvt_null) { + case mvt_null: fail = true; // null op string => null return ""; - } else { + case mvt_no_such_key: + default: fprintf(stderr, "unhandled mvt_type %d\n", one.type); exit(EXIT_IMPOSSIBLE); } } -int compare_fsl(mvt_value one, json_object *two, bool &fail) { +int compare_fsl(mvt_value const &one, json_object *two, bool &fail) { // In FSL expressions, the attribute value is coerced to the type // of the JSON literal value it is being compared to. // @@ -47,30 +50,38 @@ int compare_fsl(mvt_value one, json_object *two, bool &fail) { if (two->type == JSON_NUMBER) { double lhs; - if (one.type == mvt_string) { + switch (one.type) { + case mvt_string: { char *endptr = NULL; - const char *s = one.string_value.c_str(); + const char *s = one.c_str(); lhs = strtod(s, &endptr); if (endptr == s) { fail = true; // non-numeric-string op number => null return 0; } - } else if (one.type == mvt_float) { + } break; + case mvt_float: lhs = one.numeric_value.float_value; - } else if (one.type == mvt_double) { + break; + case mvt_double: lhs = one.numeric_value.double_value; - } else if (one.type == mvt_int) { + break; + case mvt_int: lhs = one.numeric_value.int_value; - } else if (one.type == mvt_uint) { + break; + case mvt_uint: lhs = one.numeric_value.uint_value; - } else if (one.type == mvt_sint) { + break; + case mvt_sint: lhs = one.numeric_value.sint_value; - } else if (one.type == mvt_bool) { + break; + case mvt_bool: lhs = one.numeric_value.bool_value; - } else if (one.type == mvt_null) { + break; + case mvt_null: fail = true; // null op number => null return 0; - } else { + default: fprintf(stderr, "unhandled mvt_type %d\n", one.type); exit(EXIT_IMPOSSIBLE); } @@ -93,24 +104,34 @@ int compare_fsl(mvt_value one, json_object *two, bool &fail) { if (two->type == JSON_TRUE || two->type == JSON_FALSE) { bool lhs; - if (one.type == mvt_string) { - lhs = one.string_value.size() > 0; - } else if (one.type == mvt_float) { + switch (one.type) { + case mvt_string: + lhs = one.get_string_view().size() > 0; + break; + case mvt_float: lhs = one.numeric_value.float_value != 0; - } else if (one.type == mvt_double) { + break; + case mvt_double: lhs = one.numeric_value.double_value != 0; - } else if (one.type == mvt_int) { + break; + case mvt_int: lhs = one.numeric_value.int_value != 0; - } else if (one.type == mvt_uint) { + break; + case mvt_uint: lhs = one.numeric_value.uint_value != 0; - } else if (one.type == mvt_sint) { + break; + case mvt_sint: lhs = one.numeric_value.sint_value != 0; - } else if (one.type == mvt_bool) { + break; + case mvt_bool: lhs = one.numeric_value.bool_value; - } else if (one.type == mvt_null) { + break; + case mvt_null: fail = true; // null op bool => null return 0; - } else { + + case mvt_no_such_key: + default: fprintf(stderr, "unhandled mvt_type %d\n", one.type); exit(EXIT_IMPOSSIBLE); } @@ -123,34 +144,45 @@ int compare_fsl(mvt_value one, json_object *two, bool &fail) { exit(EXIT_IMPOSSIBLE); } -int compare(mvt_value one, json_object *two, bool &fail) { - if (one.type == mvt_string) { +int compare(mvt_value const &one, json_object *two, bool &fail) { + switch (one.type) { + case mvt_string: if (two->type != JSON_STRING) { fail = true; return false; // string vs non-string } - return strcmp(one.string_value.c_str(), two->value.string.string); - } + return strcmp(one.c_str(), two->value.string.string); - if (one.type == mvt_double || one.type == mvt_float || one.type == mvt_int || one.type == mvt_uint || one.type == mvt_sint) { + case mvt_double: + case mvt_float: + case mvt_int: + case mvt_uint: + case mvt_sint: if (two->type != JSON_NUMBER) { fail = true; return false; // number vs non-number } double v; - if (one.type == mvt_double) { + switch (one.type) { + case mvt_double: v = one.numeric_value.double_value; - } else if (one.type == mvt_float) { + break; + case mvt_float: v = one.numeric_value.float_value; - } else if (one.type == mvt_int) { + break; + case mvt_int: v = one.numeric_value.int_value; - } else if (one.type == mvt_uint) { + break; + case mvt_uint: v = one.numeric_value.uint_value; - } else if (one.type == mvt_sint) { + break; + case mvt_sint: v = one.numeric_value.sint_value; - } else { + break; + case mvt_no_such_key: + default: fprintf(stderr, "Internal error: bad mvt type %d\n", one.type); exit(EXIT_IMPOSSIBLE); } @@ -162,25 +194,29 @@ int compare(mvt_value one, json_object *two, bool &fail) { } else { return 0; } - } - if (one.type == mvt_bool) { + case mvt_bool: if (two->type != JSON_TRUE && two->type != JSON_FALSE) { fail = true; return false; // bool vs non-bool } - bool b = two->type != JSON_FALSE; - return one.numeric_value.bool_value > b; - } + { + bool b = two->type != JSON_FALSE; + return one.numeric_value.bool_value > b; + } - if (one.type == mvt_null) { + case mvt_null: if (two->type != JSON_NULL) { fail = true; return false; // null vs non-null } return 0; // null equals null + + case mvt_no_such_key: + default: + break; } fprintf(stderr, "Internal error: bad mvt type %d\n", one.type); @@ -190,7 +226,7 @@ int compare(mvt_value one, json_object *two, bool &fail) { // 0: false // 1: true // -1: incomparable (sql null), treated as false in final output -static int eval(std::map const &feature, json_object *f, std::set &exclude_attributes) { +static int eval(std::function feature, json_object *f, std::set &exclude_attributes) { if (f != NULL) { if (f->type == JSON_TRUE) { return 1; @@ -236,10 +272,10 @@ static int eval(std::map const &feature, json_object *f, if (f->value.array.array[0]->type == JSON_STRING) { // if LHS of a boolean conjunction is a string, it is an attribute reference - auto ff = feature.find(std::string(f->value.array.array[0]->value.string.string)); - if (ff != feature.end()) { - if (ff->second.type == mvt_bool) { - lhs = ff->second.numeric_value.bool_value; + mvt_value ff = feature(std::string(f->value.array.array[0]->value.string.string)); + if (ff.type != mvt_no_such_key) { + if (ff.type == mvt_bool) { + lhs = ff.numeric_value.bool_value; } else { lhs = -1; // not boolean: null } @@ -280,9 +316,9 @@ static int eval(std::map const &feature, json_object *f, false)) { mvt_value lhs; lhs.type = mvt_null; // attributes that aren't found are nulls - auto ff = feature.find(std::string(f->value.array.array[0]->value.string.string)); - if (ff != feature.end()) { - lhs = ff->second; + mvt_value ff = feature(std::string(f->value.array.array[0]->value.string.string)); + if (ff.type != mvt_no_such_key) { + lhs = ff; } if (f->value.array.array[2]->type == JSON_NULL && strcmp(f->value.array.array[1]->value.string.string, "is") == 0) { @@ -341,9 +377,8 @@ static int eval(std::map const &feature, json_object *f, } } - int cmp = compare_fsl(ff->second, f->value.array.array[2], fail); + int cmp = compare_fsl(ff, f->value.array.array[2], fail); if (fail) { - printf("cast fail\n"); return -1; // null } @@ -387,7 +422,7 @@ static int eval(std::map const &feature, json_object *f, fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f)); exit(EXIT_FILTER); } - return feature.count(std::string(f->value.array.array[1]->value.string.string)) != 0; + return feature(std::string(f->value.array.array[1]->value.string.string)).type != mvt_no_such_key; } if (strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) { @@ -395,7 +430,7 @@ static int eval(std::map const &feature, json_object *f, fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f)); exit(EXIT_FILTER); } - return feature.count(std::string(f->value.array.array[1]->value.string.string)) == 0; + return feature(std::string(f->value.array.array[1]->value.string.string)).type == mvt_no_such_key; } } @@ -414,8 +449,8 @@ static int eval(std::map const &feature, json_object *f, exit(EXIT_FILTER); } - auto ff = feature.find(std::string(f->value.array.array[1]->value.string.string)); - if (ff == feature.end()) { + mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string)); + if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { const char *s = json_stringify(f); @@ -430,7 +465,7 @@ static int eval(std::map const &feature, json_object *f, } bool fail = false; - int cmp = compare(ff->second, f->value.array.array[2], fail); + int cmp = compare(ff, f->value.array.array[2], fail); if (fail) { static bool warned = false; @@ -517,8 +552,8 @@ static int eval(std::map const &feature, json_object *f, exit(EXIT_FILTER); } - auto ff = feature.find(std::string(f->value.array.array[1]->value.string.string)); - if (ff == feature.end()) { + mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string)); + if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { const char *s = json_stringify(f); @@ -535,7 +570,7 @@ static int eval(std::map const &feature, json_object *f, bool found = false; for (size_t i = 2; i < f->value.array.length; i++) { bool fail = false; - int cmp = compare(ff->second, f->value.array.array[i], fail); + int cmp = compare(ff, f->value.array.array[i], fail); if (fail) { static bool warned = false; @@ -584,7 +619,7 @@ static int eval(std::map const &feature, json_object *f, exit(EXIT_FILTER); } -bool evaluate(std::map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes) { +bool evaluate(std::function feature, std::string const &layer, json_object *filter, std::set &exclude_attributes) { if (filter == NULL || filter->type != JSON_HASH) { fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter)); exit(EXIT_JSON); @@ -638,48 +673,69 @@ json_object *parse_filter(const char *s) { return filter; } -bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set &exclude_attributes, int z) { - if (filter != NULL) { - std::map attributes; - - for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { - std::string key = layer.keys[feat.tags[t]]; - const mvt_value &val = layer.values[feat.tags[t + 1]]; - - attributes.insert(std::pair(key, val)); +bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes) { + std::function getter = [&](std::string const &key) { + auto f = feature.find(key); + if (f != feature.end()) { + return f->second; + } else { + mvt_value v; + v.type = mvt_no_such_key; + v.numeric_value.null_value = 0; + return v; } + }; - if (feat.has_id) { + return evaluate(getter, layer, filter, exclude_attributes); +} + +bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set &exclude_attributes, int z) { + std::function getter = [&](std::string const &key) { + const static std::string dollar_id = "$id"; + if (key == dollar_id && feat.has_id) { mvt_value v; v.type = mvt_uint; v.numeric_value.uint_value = feat.id; + return v; + } - attributes.insert(std::pair("$id", v)); + const static std::string dollar_type = "$type"; + if (key == dollar_type) { + mvt_value v; + v.type = mvt_string; + + if (feat.type == mvt_point) { + const static std::string point = "Point"; + v.set_string_value(point); + } else if (feat.type == mvt_linestring) { + const static std::string linestring = "LineString"; + v.set_string_value(linestring); + } else if (feat.type == mvt_polygon) { + const static std::string polygon = "Polygon"; + v.set_string_value(polygon); + } + return v; + } + + const static std::string dollar_zoom = "$zoom"; + if (key == dollar_zoom) { + mvt_value v2; + v2.type = mvt_uint; + v2.numeric_value.uint_value = z; + return v2; + } + + for (size_t i = 0; i + 1 < feat.tags.size(); i += 2) { + if (layer.keys[feat.tags[i]] == key) { + return layer.values[feat.tags[i + 1]]; + } } mvt_value v; - v.type = mvt_string; + v.type = mvt_no_such_key; + v.numeric_value.null_value = 0; + return v; + }; - if (feat.type == mvt_point) { - v.string_value = "Point"; - } else if (feat.type == mvt_linestring) { - v.string_value = "LineString"; - } else if (feat.type == mvt_polygon) { - v.string_value = "Polygon"; - } - - attributes.insert(std::pair("$type", v)); - - mvt_value v2; - v2.type = mvt_uint; - v2.numeric_value.uint_value = z; - - attributes.insert(std::pair("$zoom", v2)); - - if (!evaluate(attributes, layer.name, filter, exclude_attributes)) { - return false; - } - } - - return true; + return evaluate(getter, layer.name, filter, exclude_attributes); } diff --git a/evaluator.hpp b/evaluator.hpp index d8cc9e8b..b64fb095 100644 --- a/evaluator.hpp +++ b/evaluator.hpp @@ -1,13 +1,13 @@ #ifndef EVALUATOR_HPP #define EVALUATOR HPP -#include +#include #include #include #include "jsonpull/jsonpull.h" #include "mvt.hpp" -bool evaluate(std::map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes); +bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes); json_object *parse_filter(const char *s); json_object *read_filter(const char *fname); diff --git a/geometry.hpp b/geometry.hpp index c4fa4d85..cb764992 100644 --- a/geometry.hpp +++ b/geometry.hpp @@ -100,16 +100,16 @@ void visvalingam(drawvec &ls, size_t start, size_t end, double threshold, size_t int pnpoly(const drawvec &vert, size_t start, size_t nvert, long long testx, long long testy); double distance_from_line(long long point_x, long long point_y, long long segA_x, long long segA_y, long long segB_x, long long segB_y); -std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny, +std::string overzoom(const mvt_tile &tile, int oz, int ox, int oy, int nz, int nx, int ny, int detail, int buffer, std::set const &keep, bool do_compress, std::vector> *next_overzoomed_tiles, bool demultiply, json_object *filter, bool preserve_input_order, - std::map const &attribute_accum); + std::unordered_map const &attribute_accum); -std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny, +std::string overzoom(const std::string &s, int oz, int ox, int oy, int nz, int nx, int ny, int detail, int buffer, std::set const &keep, bool do_compress, std::vector> *next_overzoomed_tiles, bool demultiply, json_object *filter, bool preserve_input_order, - std::map const &attribute_accum); + std::unordered_map const &attribute_accum); #endif diff --git a/main.cpp b/main.cpp index c0e72f60..3f065448 100644 --- a/main.cpp +++ b/main.cpp @@ -448,7 +448,7 @@ void *run_sort(void *v) { return NULL; } -void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector *readers, std::atomic *progress_seq, std::set *exclude, std::set *include, int exclude_all, int basezoom, int source, std::vector > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map const *attribute_types, int separator, double *dist_sum, size_t *dist_count, double *area_sum, bool want_dist, bool filters) { +void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector *readers, std::atomic *progress_seq, std::set *exclude, std::set *include, int exclude_all, int basezoom, int source, std::vector > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::unordered_map const *attribute_types, int separator, double *dist_sum, size_t *dist_count, double *area_sum, bool want_dist, bool filters) { long long segs[CPUS + 1]; segs[0] = 0; segs[CPUS] = len; @@ -649,7 +649,7 @@ struct read_parallel_arg { unsigned *initial_y = NULL; std::string layername = ""; bool uses_gamma = false; - std::map const *attribute_types = NULL; + std::unordered_map const *attribute_types = NULL; double *dist_sum = NULL; size_t *dist_count = NULL; double *area_sum = NULL; @@ -693,7 +693,7 @@ void *run_read_parallel(void *v) { return NULL; } -void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::atomic *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector *readers, std::atomic *progress_seq, std::set *exclude, std::set *include, int exclude_all, int basezoom, int source, std::vector > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map const *attribute_types, int separator, double *dist_sum, size_t *dist_count, double *area_sum, bool want_dist, bool filters) { +void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::atomic *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector *readers, std::atomic *progress_seq, std::set *exclude, std::set *include, int exclude_all, int basezoom, int source, std::vector > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::unordered_map const *attribute_types, int separator, double *dist_sum, size_t *dist_count, double *area_sum, bool want_dist, bool filters) { // This has to kick off an intermediate thread to start the parser threads, // so the main thread can get back to reading the next input stage while // the intermediate thread waits for the completion of the parser threads. @@ -1237,7 +1237,7 @@ int vertexcmp(const void *void1, const void *void2) { return 0; } -std::pair read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::map const *attribute_types, const char *pgm, std::map const *attribute_accum, std::map const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) { +std::pair read_input(std::vector &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set *exclude, std::set *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::unordered_map const *attribute_types, const char *pgm, std::unordered_map const *attribute_accum, std::map const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) { int ret = EXIT_SUCCESS; std::vector readers; @@ -2845,7 +2845,7 @@ static bool has_name(struct option *long_options, int *pl) { return false; } -void set_attribute_type(std::map &attribute_types, const char *arg) { +void set_attribute_type(std::unordered_map &attribute_types, const char *arg) { const char *s = strchr(arg, ':'); if (s == NULL) { fprintf(stderr, "-T%s option must be in the form -Tname:type\n", arg); @@ -3005,8 +3005,8 @@ int main(int argc, char **argv) { bool guess_cluster_maxzoom = false; std::set exclude, include; - std::map attribute_types; - std::map attribute_accum; + std::unordered_map attribute_types; + std::unordered_map attribute_accum; std::map attribute_descriptions; int exclude_all = 0; int read_parallel = 0; diff --git a/mvt.cpp b/mvt.cpp index b693a717..16481015 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -110,7 +110,7 @@ int compress(std::string const &input, std::string &output, bool gz) { return 0; } -bool mvt_tile::decode(std::string &message, bool &was_compressed) { +bool mvt_tile::decode(const std::string &message, bool &was_compressed) { layers.clear(); std::string src; @@ -127,6 +127,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { } protozero::pbf_reader reader(src); + std::shared_ptr string_pool = std::make_shared(); while (reader.next()) { switch (reader.tag()) { @@ -157,7 +158,12 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { switch (value_reader.tag()) { case 1: /* string */ value.type = mvt_string; - value.string_value = value_reader.get_string(); + value.s = string_pool; + { + auto v = value_reader.get_view(); + std::string_view sv(v.data(), v.size()); + value.set_string_value(sv); + } break; case 2: /* float */ @@ -196,7 +202,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { } } - layer.values.push_back(value); + layer.values.push_back(std::move(value)); break; } @@ -224,6 +230,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { case 2: /* tag */ { auto pi = feature_reader.get_packed_uint32(); + feature.tags.reserve(std::distance(pi.first, pi.second)); for (auto it = pi.first; it != pi.second; ++it) { feature.tags.push_back(*it); } @@ -237,6 +244,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { case 4: /* geometry */ { auto pi = feature_reader.get_packed_uint32(); + geoms.reserve(std::distance(pi.first, pi.second)); for (auto it = pi.first; it != pi.second; ++it) { geoms.push_back(*it); } @@ -250,6 +258,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { } long long px = 0, py = 0; + feature.geometry.reserve(geoms.size()); // probably not quite right, but still plausible for (size_t g = 0; g < geoms.size(); g++) { uint32_t geom = geoms[g]; uint32_t op = geom & 7; @@ -261,14 +270,14 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { py += protozero::decode_zigzag32(geoms[g + 2]); g += 2; - feature.geometry.push_back(mvt_geometry(op, px, py)); + feature.geometry.emplace_back(op, px, py); } } else { - feature.geometry.push_back(mvt_geometry(op, 0, 0)); + feature.geometry.emplace_back(op, 0, 0); } } - layer.features.push_back(feature); + layer.features.push_back(std::move(feature)); break; } @@ -278,14 +287,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) { } } - for (size_t i = 0; i < layer.keys.size(); i++) { - layer.key_map.insert(std::pair(layer.keys[i], i)); - } - for (size_t i = 0; i < layer.values.size(); i++) { - layer.value_map.insert(std::pair(layer.values[i], i)); - } - - layers.push_back(layer); + layers.push_back(std::move(layer)); break; } @@ -313,7 +315,6 @@ struct sorted_value { std::string mvt_tile::encode() { std::string data; - protozero::pbf_writer writer(data); for (size_t i = 0; i < layers.size(); i++) { @@ -335,32 +336,40 @@ std::string mvt_tile::encode() { protozero::pbf_writer value_writer(value_string); mvt_value &pbv = layers[i].values[v]; - if (pbv.type == mvt_string) { - value_writer.add_string(1, pbv.string_value); - } else if (pbv.type == mvt_float) { + switch (pbv.type) { + case mvt_string: + value_writer.add_string(1, pbv.get_string_value()); + break; + case mvt_float: value_writer.add_float(2, pbv.numeric_value.float_value); - } else if (pbv.type == mvt_double) { + break; + case mvt_double: value_writer.add_double(3, pbv.numeric_value.double_value); - } else if (pbv.type == mvt_int) { + break; + case mvt_int: value_writer.add_int64(4, pbv.numeric_value.int_value); - } else if (pbv.type == mvt_uint) { + break; + case mvt_uint: value_writer.add_uint64(5, pbv.numeric_value.uint_value); - } else if (pbv.type == mvt_sint) { + break; + case mvt_sint: value_writer.add_sint64(6, pbv.numeric_value.sint_value); - } else if (pbv.type == mvt_bool) { + break; + case mvt_bool: value_writer.add_bool(7, pbv.numeric_value.bool_value); - } else if (pbv.type == mvt_null) { + break; + case mvt_null: fprintf(stderr, "Internal error: trying to write null attribute to tile\n"); exit(EXIT_IMPOSSIBLE); - } else { + default: fprintf(stderr, "Internal error: trying to write undefined attribute type to tile\n"); exit(EXIT_IMPOSSIBLE); } sorted_value sv; - sv.val = value_string; + sv.val = std::move(value_string); sv.orig = v; - sorted_values.push_back(sv); + sorted_values.push_back(std::move(sv)); } std::sort(sorted_values.begin(), sorted_values.end()); @@ -456,15 +465,70 @@ bool mvt_value::operator<(const mvt_value &o) const { return true; } if (type == o.type) { - if ((type == mvt_string && string_value < o.string_value) || - (type == mvt_float && numeric_value.float_value < o.numeric_value.float_value) || - (type == mvt_double && numeric_value.double_value < o.numeric_value.double_value) || - (type == mvt_int && numeric_value.int_value < o.numeric_value.int_value) || - (type == mvt_uint && numeric_value.uint_value < o.numeric_value.uint_value) || - (type == mvt_sint && numeric_value.sint_value < o.numeric_value.sint_value) || - (type == mvt_bool && numeric_value.bool_value < o.numeric_value.bool_value) || - (type == mvt_null && numeric_value.null_value < o.numeric_value.null_value)) { - return true; + switch (type) { + case mvt_string: + return get_string_view() < o.get_string_view(); + + case mvt_float: + return numeric_value.float_value < o.numeric_value.float_value; + + case mvt_double: + return numeric_value.double_value < o.numeric_value.double_value; + + case mvt_int: + return numeric_value.int_value < o.numeric_value.int_value; + + case mvt_uint: + return numeric_value.uint_value < o.numeric_value.uint_value; + + case mvt_sint: + return numeric_value.sint_value < o.numeric_value.sint_value; + + case mvt_bool: + return numeric_value.bool_value < o.numeric_value.bool_value; + + case mvt_null: + return numeric_value.null_value < o.numeric_value.null_value; + + default: + fprintf(stderr, "mvt_value::operator<<: can't happen\n"); + exit(EXIT_IMPOSSIBLE); + } + } + + return false; +} + +bool mvt_value::operator==(const mvt_value &o) const { + if (type == o.type) { + switch (type) { + case mvt_string: + return get_string_view() == o.get_string_view(); + + case mvt_float: + return numeric_value.float_value == o.numeric_value.float_value; + + case mvt_double: + return numeric_value.double_value == o.numeric_value.double_value; + + case mvt_int: + return numeric_value.int_value == o.numeric_value.int_value; + + case mvt_uint: + return numeric_value.uint_value == o.numeric_value.uint_value; + + case mvt_sint: + return numeric_value.sint_value == o.numeric_value.sint_value; + + case mvt_bool: + return numeric_value.bool_value == o.numeric_value.bool_value; + + case mvt_null: + return numeric_value.null_value == o.numeric_value.null_value; + + default: + fprintf(stderr, "mvt_value::operator==: can't happen\n"); + exit(EXIT_IMPOSSIBLE); } } @@ -493,47 +557,62 @@ static std::string quote(std::string const &s) { } std::string mvt_value::toString() const { - if (type == mvt_string) { - return quote(string_value); - } else if (type == mvt_int) { + switch (type) { + case mvt_string: + return quote(get_string_value()); + case mvt_int: return std::to_string(numeric_value.int_value); - } else if (type == mvt_double) { + case mvt_double: { double v = numeric_value.double_value; if (v == (long long) v) { return std::to_string((long long) v); } else { return milo::dtoa_milo(v); } - } else if (type == mvt_float) { + } + case mvt_float: { double v = numeric_value.float_value; if (v == (long long) v) { return std::to_string((long long) v); } else { return milo::dtoa_milo(v); } - } else if (type == mvt_sint) { + } + case mvt_sint: return std::to_string(numeric_value.sint_value); - } else if (type == mvt_uint) { + case mvt_uint: return std::to_string(numeric_value.uint_value); - } else if (type == mvt_bool) { + case mvt_bool: return numeric_value.bool_value ? "true" : "false"; - } else if (type == mvt_null) { + case mvt_null: return "null"; - } else { + default: return "unknown"; } } -void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) { +void mvt_layer::tag(mvt_feature &feature, std::string const &key, mvt_value const &value) { size_t ko, vo; - std::map::iterator ki = key_map.find(key); - std::map::iterator vi = value_map.find(value); + // initialize lazily the first time anyone tags an attribute + // to save the time of doing it in decode, which never actually matters. + // only tile writers actually need this. + if (key_map.size() == 0) { + for (size_t i = 0; i < keys.size(); i++) { + key_map.emplace(keys[i], i); + } + for (size_t i = 0; i < values.size(); i++) { + value_map.emplace(values[i], i); + } + } + + std::unordered_map::iterator ki = key_map.find(key); + std::unordered_map::iterator vi = value_map.find(value); if (ki == key_map.end()) { ko = keys.size(); keys.push_back(key); - key_map.insert(std::pair(key, ko)); + key_map.emplace(key, ko); } else { ko = ki->second; } @@ -541,7 +620,7 @@ void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) { if (vi == value_map.end()) { vo = values.size(); values.push_back(value); - value_map.insert(std::pair(value, vo)); + value_map.emplace(value, vo); } else { vo = vi->second; } @@ -629,7 +708,8 @@ bool is_unsigned_integer(const char *s, unsigned long long *v) { mvt_value stringified_to_mvt_value(int type, const char *s) { mvt_value tv; - if (type == mvt_double) { + switch (type) { + case mvt_double: { long long v; unsigned long long uv; if (is_unsigned_integer(s, &uv)) { @@ -668,15 +748,18 @@ mvt_value stringified_to_mvt_value(int type, const char *s) { } } } - } else if (type == mvt_bool) { + } break; + case mvt_bool: tv.type = mvt_bool; tv.numeric_value.bool_value = (s[0] == 't'); - } else if (type == mvt_null) { + break; + case mvt_null: tv.type = mvt_null; tv.numeric_value.null_value = 0; - } else { + break; + default: tv.type = mvt_string; - tv.string_value = s; + tv.set_string_value(s); } return tv; @@ -690,34 +773,68 @@ mvt_value stringified_to_mvt_value(int type, const char *s) { serial_val mvt_value_to_serial_val(mvt_value const &v) { serial_val sv; - if (v.type == mvt_string) { + switch (v.type) { + case mvt_string: sv.type = mvt_string; - sv.s = v.string_value; - } else if (v.type == mvt_float) { + sv.s = v.get_string_value(); + break; + case mvt_float: sv.type = mvt_double; sv.s = milo::dtoa_milo(v.numeric_value.float_value); - } else if (v.type == mvt_double) { + break; + case mvt_double: sv.type = mvt_double; sv.s = milo::dtoa_milo(v.numeric_value.double_value); - } else if (v.type == mvt_int) { + break; + case mvt_int: sv.type = mvt_double; sv.s = std::to_string(v.numeric_value.int_value); - } else if (v.type == mvt_uint) { + break; + case mvt_uint: sv.type = mvt_double; sv.s = std::to_string(v.numeric_value.uint_value); - } else if (v.type == mvt_sint) { + break; + case mvt_sint: sv.type = mvt_double; sv.s = std::to_string(v.numeric_value.sint_value); - } else if (v.type == mvt_bool) { + break; + case mvt_bool: sv.type = mvt_bool; sv.s = v.numeric_value.bool_value ? "true" : "false"; - } else if (v.type == mvt_null) { + break; + case mvt_null: sv.type = mvt_null; sv.s = "null"; - } else { + break; + default: fprintf(stderr, "unhandled mvt_type %d\n", v.type); exit(EXIT_IMPOSSIBLE); } return sv; } + +// This extracts an integer value from an mvt_value +long long mvt_value_to_long_long(mvt_value const &v) { + switch (v.type) { + case mvt_string: + return atoll(v.c_str()); + case mvt_float: + return v.numeric_value.float_value; + case mvt_double: + return v.numeric_value.double_value; + case mvt_int: + return v.numeric_value.int_value; + case mvt_uint: + return v.numeric_value.uint_value; + case mvt_sint: + return v.numeric_value.sint_value; + case mvt_bool: + return v.numeric_value.bool_value; + case mvt_null: + return 0; + default: + fprintf(stderr, "unhandled mvt_type %d\n", v.type); + exit(EXIT_IMPOSSIBLE); + } +} diff --git a/mvt.hpp b/mvt.hpp index c848cf6f..ba0d5d57 100644 --- a/mvt.hpp +++ b/mvt.hpp @@ -3,9 +3,14 @@ #include #include -#include +#include +#include #include #include +#include +#include + +#include "errors.hpp" struct mvt_value; struct mvt_layer; @@ -66,11 +71,14 @@ enum mvt_value_type { mvt_sint, mvt_bool, mvt_null, + + mvt_no_such_key, }; struct mvt_value { mvt_value_type type; - std::string string_value; + std::shared_ptr s; + union { float float_value; double double_value; @@ -79,18 +87,81 @@ struct mvt_value { long long sint_value; bool bool_value; int null_value; + struct { + size_t off; + size_t len; + } string_value; } numeric_value; + std::string get_string_value() const { + return std::string(*s, numeric_value.string_value.off, numeric_value.string_value.len); + } + + std::string_view get_string_view() const { + return std::string_view(s->c_str() + numeric_value.string_value.off, numeric_value.string_value.len); + } + + const char *c_str() const { + return s->c_str() + numeric_value.string_value.off; + } + + void set_string_value(const std::string_view &val) { + if (s == nullptr) { + s = std::make_shared(); + } + + type = mvt_string; + numeric_value.string_value.off = s->size(); + numeric_value.string_value.len = val.size(); + s->append(val); + s->push_back('\0'); + } + bool operator<(const mvt_value &o) const; + bool operator==(const mvt_value &o) const; std::string toString() const; mvt_value() { this->type = mvt_double; - this->string_value = ""; this->numeric_value.double_value = 0; } }; +template <> +struct std::hash { + std::size_t operator()(const mvt_value &k) const { + switch (k.type) { + case mvt_string: + return std::hash()(k.get_string_view()); + + case mvt_float: + return std::hash()(k.numeric_value.float_value); + + case mvt_double: + return std::hash()(k.numeric_value.double_value); + + case mvt_int: + return std::hash()(k.numeric_value.int_value); + + case mvt_uint: + return std::hash()(k.numeric_value.uint_value); + + case mvt_sint: + return std::hash()(k.numeric_value.sint_value); + + case mvt_bool: + return std::hash()(k.numeric_value.bool_value); + + case mvt_null: + return std::hash()(k.numeric_value.null_value); + + default: + fprintf(stderr, "mvt_value hash can't happen\n"); + exit(EXIT_IMPOSSIBLE); + } + } +}; + struct mvt_layer { int version = 0; std::string name = ""; @@ -100,18 +171,18 @@ struct mvt_layer { long long extent = 0; // Add a key-value pair to a feature, using this layer's constant pool - void tag(mvt_feature &feature, std::string key, mvt_value value); + void tag(mvt_feature &feature, std::string const &key, mvt_value const &value); // For tracking the key-value constants already used in this layer - std::map key_map{}; - std::map value_map{}; + std::unordered_map key_map{}; + std::unordered_map value_map{}; }; struct mvt_tile { std::vector layers{}; std::string encode(); - bool decode(std::string &message, bool &was_compressed); + bool decode(const std::string &message, bool &was_compressed); }; bool is_compressed(std::string const &data); @@ -120,6 +191,7 @@ int compress(std::string const &input, std::string &output, bool gz); int dezig(unsigned n); mvt_value stringified_to_mvt_value(int type, const char *s); +long long mvt_value_to_long_long(mvt_value const &v); bool is_integer(const char *s, long long *v); bool is_unsigned_integer(const char *s, unsigned long long *v); diff --git a/overzoom.cpp b/overzoom.cpp index 1c9d365e..f1033d02 100644 --- a/overzoom.cpp +++ b/overzoom.cpp @@ -17,7 +17,7 @@ int buffer = 5; // tippecanoe-style: mvt buffer == extent * buffer / 256; bool demultiply = false; std::string filter; bool preserve_input_order = false; -std::map attribute_accum; +std::unordered_map attribute_accum; std::set keep; diff --git a/pmtiles/pmtiles.hpp b/pmtiles/pmtiles.hpp index 01a1a19f..98c77730 100644 --- a/pmtiles/pmtiles.hpp +++ b/pmtiles/pmtiles.hpp @@ -166,11 +166,11 @@ struct entryv3 { } }; -struct { +struct entryv3_cmp { bool operator()(entryv3 a, entryv3 b) const { return a.tile_id < b.tile_id; } -} entryv3_cmp; +}; struct entry_zxy { uint8_t z; @@ -388,7 +388,7 @@ inline uint64_t zxy_to_tileid(uint8_t z, uint32_t x, uint32_t y) { if (z > 31) { throw std::overflow_error("tile zoom exceeds 64-bit limit"); } - if (x > (1 << z) - 1 || y > (1 << z) - 1) { + if (x > (1U << z) - 1 || y > (1U << z) - 1) { throw std::overflow_error("tile x/y outside zoom level bounds"); } uint64_t acc = 0; diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index 0326c92a..5cd78620 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -263,7 +263,7 @@ void mbtiles_map_image_to_pmtiles(char *fname, metadata m, bool tile_compression // finalize PMTiles archive. { - std::sort(entries.begin(), entries.end(), pmtiles::entryv3_cmp); + std::sort(entries.begin(), entries.end(), pmtiles::entryv3_cmp()); std::string root_bytes; std::string leaves_bytes; diff --git a/serial.cpp b/serial.cpp index cbb974a8..d7505ee6 100644 --- a/serial.cpp +++ b/serial.cpp @@ -878,7 +878,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { return 1; } -void coerce_value(std::string const &key, int &vt, std::string &val, std::map const *attribute_types) { +void coerce_value(std::string const &key, int &vt, std::string &val, std::unordered_map const *attribute_types) { auto a = (*attribute_types).find(key); if (a != attribute_types->end()) { if (a->second == mvt_string) { diff --git a/serial.hpp b/serial.hpp index 05e6d4ea..fecbdf41 100644 --- a/serial.hpp +++ b/serial.hpp @@ -175,7 +175,7 @@ struct serialization_state { std::map *layermap = NULL; - std::map const *attribute_types = NULL; + std::unordered_map const *attribute_types = NULL; std::set *exclude = NULL; std::set *include = NULL; int exclude_all = 0; @@ -228,6 +228,6 @@ struct node { int nodecmp(const void *void1, const void *void2); int serialize_feature(struct serialization_state *sst, serial_feature &sf); -void coerce_value(std::string const &key, int &vt, std::string &val, std::map const *attribute_types); +void coerce_value(std::string const &key, int &vt, std::string &val, std::unordered_map const *attribute_types); #endif diff --git a/tile-join.cpp b/tile-join.cpp index 8492e8f7..f24983c4 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -140,7 +140,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map exclude_attributes; - if (!evaluate(feat, layer, filter, exclude_attributes, z)) { + if (filter != NULL && !evaluate(feat, layer, filter, exclude_attributes, z)) { continue; } @@ -197,7 +197,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map getter = [&](zxy tile) { - return get_tile(tile); + std::function getter = [&](zxy tileno) { + return get_tile(tileno); }; mvt_tile source = cache.get(parent_tile, getter); @@ -699,7 +699,7 @@ struct tileset_reader { } if (source.layers.size() != 0) { - std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set(), false, &next_overzoomed_tiles, false, NULL, false, std::map()); + std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set(), false, &next_overzoomed_tiles, false, NULL, false, std::unordered_map()); return ret; } diff --git a/tile.cpp b/tile.cpp index 68c36bc4..e763137e 100644 --- a/tile.cpp +++ b/tile.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -216,7 +217,7 @@ void decode_meta(std::vector const &metakeys, std::vector mvt_value key = retrieve_string(metakeys[i], stringpool, NULL); mvt_value value = retrieve_string(metavals[i], stringpool, &otype); - layer.tag(feature, key.string_value, value); + layer.tag(feature, key.get_string_value(), value); } } @@ -226,9 +227,9 @@ static int metacmp(const std::vector &keys1, const std::vector key2.string_value) { + } else if (key1.get_string_view() > key2.get_string_view()) { return 1; } @@ -258,7 +259,7 @@ static int metacmp(const std::vector &keys1, const std::vector need_tilestats; - std::map attribute_accum_state; + std::unordered_map attribute_accum_state; }; struct partial_arg { @@ -1404,7 +1405,7 @@ struct write_tile_args { size_t feature_count_out = 0; const char *prefilter = NULL; const char *postfilter = NULL; - std::map const *attribute_accum = NULL; + std::unordered_map const *attribute_accum = NULL; bool still_dropping = false; int wrote_zoom = 0; size_t tiling_seg = 0; @@ -1593,7 +1594,7 @@ serial_feature next_feature(decompressor *geoms, std::atomic *geompos } if (filter != NULL) { - std::map attributes; + std::unordered_map attributes; std::string layername = (*layer_unmaps)[sf.segment][sf.layer]; std::set exclude_attributes; @@ -1627,11 +1628,11 @@ serial_feature next_feature(decompressor *geoms, std::atomic *geompos v.type = mvt_string; if (sf.t == mvt_point) { - v.string_value = "Point"; + v.set_string_value("Point"); } else if (sf.t == mvt_linestring) { - v.string_value = "LineString"; + v.set_string_value("LineString"); } else if (sf.t == mvt_polygon) { - v.string_value = "Polygon"; + v.set_string_value("Polygon"); } attributes.insert(std::pair("$type", v)); @@ -1846,7 +1847,7 @@ void promote_attribute(std::string const &key, partial &p, char *stringpool, lon } } -void preserve_attributes(std::map const *attribute_accum, serial_feature &sf, char *stringpool, long long *pool_off, partial &p) { +void preserve_attributes(std::unordered_map const *attribute_accum, serial_feature &sf, char *stringpool, long long *pool_off, partial &p) { for (size_t i = 0; i < sf.keys.size(); i++) { std::string key = stringpool + pool_off[sf.segment] + sf.keys[i] + 1; @@ -3117,7 +3118,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos) { last_progress = 0; // The existing layermaps are one table per input thread. diff --git a/tile.hpp b/tile.hpp index 6155df0b..ede9d2ef 100644 --- a/tile.hpp +++ b/tile.hpp @@ -55,7 +55,7 @@ struct strategy { // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/write_json.cpp b/write_json.cpp index b5f37670..02fbec9d 100644 --- a/write_json.cpp +++ b/write_json.cpp @@ -336,31 +336,40 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y const char *key = layer.keys[feat.tags[t]].c_str(); mvt_value const &val = layer.values[feat.tags[t + 1]]; - if (val.type == mvt_string) { + switch (val.type) { + case mvt_string: state.json_write_string(key); - state.json_write_string(val.string_value); - } else if (val.type == mvt_int) { + state.json_write_string(val.get_string_value()); + break; + case mvt_int: state.json_write_string(key); state.json_write_signed(val.numeric_value.int_value); - } else if (val.type == mvt_double) { + break; + case mvt_double: state.json_write_string(key); state.json_write_number(val.numeric_value.double_value); - } else if (val.type == mvt_float) { + break; + case mvt_float: state.json_write_string(key); state.json_write_number(val.numeric_value.float_value); - } else if (val.type == mvt_sint) { + break; + case mvt_sint: state.json_write_string(key); state.json_write_signed(val.numeric_value.sint_value); - } else if (val.type == mvt_uint) { + break; + case mvt_uint: state.json_write_string(key); state.json_write_unsigned(val.numeric_value.uint_value); - } else if (val.type == mvt_bool) { + break; + case mvt_bool: state.json_write_string(key); state.json_write_bool(val.numeric_value.bool_value); - } else if (val.type == mvt_null) { + break; + case mvt_null: state.json_write_string(key); state.json_write_null(); - } else { + break; + default: fprintf(stderr, "Internal error: property with unknown type\n"); exit(EXIT_IMPOSSIBLE); }