diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Speeding up tippecanoe-overzoom (#191)

Browse Source 
* Speed up mvt_value comparison

* Converting repetitive ifs to cases

* More conversions from ifs to cases

* Optimize the always-true filter case

* Don't convert types of attributes without accumulators

* Unordered map seems to be faster than map

* Add missing header

* Fix some warnings

* Fix the warnings better

* Avoid an int->string->int conversion

* Lazily initialize layer key and values maps when actually needed

* More switches from maps to unordered_maps

* Sure, I'll take the microoptimization

* More emplacement

* Save some copies

* Emplaces and moves

* Lazy linear scan of attributes instead of building a map

* Extra printfs, missing header

* Avoid clipping if the input and output tiles are the same

* But do clip if the tile extent is being reduced

* Make sure I'm not constructing std::strings here at runtime

* More worrying about runtime string construction

* A couple more std::moves

* Const references!

* More const references

* Another std::move

* Make the string_value of mvt_value std::optional

* Reserve storage when decoding

* Provision for different mvt_values to share a string pool

* Use the string pool when decoding

* Avoid another string construction

* Try limiting the depth of the search for duplicate attributes

* Revert "Try limiting the depth of the search for duplicate attributes"

This reverts commit 9ec94a15ff.

* Update changelog

* Fix typo noticed during code review
This commit is contained in:
Erica Fischer
2024-01-29 11:33:56 -08:00
committed by GitHub
parent cbf222754b
commit 7d4d264d50
20 changed files with 553 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+2
View File
@@ -1,3 +1,5 @@
* Performance optimizations to tile reading, writing, and overzooming
# 2.41.2 # 2.41.2
* Add --accumulate-attribute to tippecanoe-overzoom * Add --accumulate-attribute to tippecanoe-overzoom
attribute.cpp
+4 -4
View File
@@ -1,12 +1,12 @@
#include <string> #include <string>
#include <map> #include <unordered_map>
#include "attribute.hpp" #include "attribute.hpp"
#include "errors.hpp" #include "errors.hpp"
#include "serial.hpp" #include "serial.hpp"
#include "jsonpull/jsonpull.h" #include "jsonpull/jsonpull.h"
#include "milo/dtoa_milo.h" #include "milo/dtoa_milo.h"
void set_attribute_accum(std::map<std::string, attribute_op> &attribute_accum, std::string name, std::string type) { void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, std::string name, std::string type) {
attribute_op t; attribute_op t;
if (type == "sum") { if (type == "sum") {
@@ -31,7 +31,7 @@ void set_attribute_accum(std::map<std::string, attribute_op> &attribute_accum, s
attribute_accum.insert(std::pair<std::string, attribute_op>(name, t)); attribute_accum.insert(std::pair<std::string, attribute_op>(name, t));
} }
void set_attribute_accum(std::map<std::string, attribute_op> &attribute_accum, const char *arg, char **argv) { void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, const char *arg, char **argv) {
if (*arg == '{') { if (*arg == '{') {
json_pull *jp = json_begin_string(arg); json_pull *jp = json_begin_string(arg);
json_object *o = json_read_tree(jp); json_object *o = json_read_tree(jp);
@@ -79,7 +79,7 @@ void set_attribute_accum(std::map<std::string, attribute_op> &attribute_accum, c
set_attribute_accum(attribute_accum, name, type); set_attribute_accum(attribute_accum, name, type);
} }
void preserve_attribute(attribute_op op, std::string &key, serial_val &val, std::vector<std::string> &full_keys, std::vector<serial_val> &full_values, std::map<std::string, accum_state> &attribute_accum_state) { void preserve_attribute(attribute_op op, std::string &key, serial_val &val, std::vector<std::string> &full_keys, std::vector<serial_val> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state) {
for (size_t i = 0; i < full_keys.size(); i++) { for (size_t i = 0; i < full_keys.size(); i++) {
if (key == full_keys[i]) { if (key == full_keys[i]) {
switch (op) { switch (op) {
attribute.hpp
+4 -4
View File
@@ -2,7 +2,7 @@
#define ATTRIBUTE_HPP #define ATTRIBUTE_HPP
#include <vector> #include <vector>
#include <map> #include <unordered_map>
enum attribute_op { enum attribute_op {
op_sum, op_sum,
@@ -21,8 +21,8 @@ struct accum_state {
struct serial_val; struct serial_val;
void set_attribute_accum(std::map<std::string, attribute_op> &attribute_accum, std::string name, std::string type); void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, std::string name, std::string type);
void set_attribute_accum(std::map<std::string, attribute_op> &attribute_accum, const char *arg, char **argv); void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, const char *arg, char **argv);
void preserve_attribute(attribute_op op, std::string &key, serial_val &val, std::vector<std::string> &full_keys, std::vector<serial_val> &full_values, std::map<std::string, accum_state> &attribute_accum_state); void preserve_attribute(attribute_op op, std::string &key, serial_val &val, std::vector<std::string> &full_keys, std::vector<serial_val> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state);
#endif #endif
clip.cpp
+61 -41
View File
@@ -755,10 +755,10 @@ static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<d
return out; return out;
} }
std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny, std::string overzoom(const std::string &s, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress, int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles, std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
bool demultiply, json_object *filter, bool preserve_input_order, std::map<std::string, attribute_op> const &attribute_accum) { bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map<std::string, attribute_op> const &attribute_accum) {
mvt_tile tile; mvt_tile tile;
try { try {
@@ -785,7 +785,7 @@ struct tile_feature {
size_t seq = 0; size_t seq = 0;
}; };
static void feature_out(std::vector<tile_feature> const &features, mvt_layer &outlayer, std::set<std::string> const &keep, std::map<std::string, attribute_op> const &attribute_accum) { static void feature_out(std::vector<tile_feature> const &features, mvt_layer &outlayer, std::set<std::string> const &keep, std::unordered_map<std::string, attribute_op> const &attribute_accum) {
// Add geometry to output feature // Add geometry to output feature
mvt_feature outfeature; mvt_feature outfeature;
@@ -810,13 +810,22 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
// attributes from the other features of the // attributes from the other features of the
// multiplier cluster accumulated onto them // multiplier cluster accumulated onto them
std::map<std::string, accum_state> attribute_accum_state; std::unordered_map<std::string, accum_state> attribute_accum_state;
std::vector<std::string> full_keys; std::vector<std::string> full_keys;
std::vector<serial_val> full_values; std::vector<serial_val> full_values;
for (size_t i = 0; i + 1 < features[0].tags.size(); i += 2) { for (size_t i = 0; i + 1 < features[0].tags.size(); i += 2) {
full_keys.push_back(features[0].layer->keys[features[0].tags[i]]); auto f = attribute_accum.find(features[0].layer->keys[features[0].tags[i]]);
full_values.push_back(mvt_value_to_serial_val(features[0].layer->values[features[0].tags[i + 1]])); if (f != attribute_accum.end()) {
// this attribute has an accumulator, so convert it
full_keys.push_back(features[0].layer->keys[features[0].tags[i]]);
full_values.push_back(mvt_value_to_serial_val(features[0].layer->values[features[0].tags[i + 1]]));
} else {
// otherwise just tag it directly onto the output feature
if (keep.size() == 0 || keep.find(features[0].layer->keys[features[0].tags[i]]) != keep.end()) {
outlayer.tag(outfeature, features[0].layer->keys[features[0].tags[i]], features[0].layer->values[features[0].tags[i + 1]]);
}
}
} }
// accumulate whatever attributes are specified to be accumulated // accumulate whatever attributes are specified to be accumulated
@@ -851,7 +860,7 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
} }
} }
outlayer.features.push_back(outfeature); outlayer.features.push_back(std::move(outfeature));
} }
} }
@@ -861,10 +870,10 @@ static struct preservecmp {
} }
} preservecmp; } preservecmp;
std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny, std::string overzoom(const mvt_tile &tile, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress, int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles, std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
bool demultiply, json_object *filter, bool preserve_input_order, std::map<std::string, attribute_op> const &attribute_accum) { bool demultiply, json_object *filter, bool preserve_input_order, std::unordered_map<std::string, attribute_op> const &attribute_accum) {
mvt_tile outtile; mvt_tile outtile;
for (auto const &layer : tile.layers) { for (auto const &layer : tile.layers) {
@@ -881,11 +890,14 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
std::vector<tile_feature> pending_tile_features; std::vector<tile_feature> pending_tile_features;
static const std::string retain_points_multiplier_first = "tippecanoe:retain_points_multiplier_first";
static const std::string retain_points_multiplier_sequence = "tippecanoe:retain_points_multiplier_sequence";
for (auto feature : layer.features) { for (auto feature : layer.features) {
bool flush_multiplier_cluster = false; bool flush_multiplier_cluster = false;
if (demultiply) { if (demultiply) {
for (ssize_t i = feature.tags.size() - 2; i >= 0; i -= 2) { for (ssize_t i = feature.tags.size() - 2; i >= 0; i -= 2) {
if (layer.keys[feature.tags[i]] == "tippecanoe:retain_points_multiplier_first") { if (layer.keys[feature.tags[i]] == retain_points_multiplier_first) {
mvt_value v = layer.values[feature.tags[i + 1]]; mvt_value v = layer.values[feature.tags[i + 1]];
if (v.type == mvt_bool && v.numeric_value.bool_value) { if (v.type == mvt_bool && v.numeric_value.bool_value) {
flush_multiplier_cluster = true; flush_multiplier_cluster = true;
@@ -893,9 +905,9 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
} }
} }
if (layer.keys[feature.tags[i]] == "tippecanoe:retain_points_multiplier_sequence") { if (layer.keys[feature.tags[i]] == retain_points_multiplier_sequence) {
mvt_value v = layer.values[feature.tags[i + 1]]; mvt_value v = layer.values[feature.tags[i + 1]];
feature.seq = atoll(mvt_value_to_serial_val(v).s.c_str()); feature.seq = mvt_value_to_long_long(v);
feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2); feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
} }
} }
@@ -911,7 +923,7 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
} }
std::set<std::string> exclude_attributes; std::set<std::string> exclude_attributes;
if (!evaluate(feature, layer, filter, exclude_attributes, nz)) { if (filter != NULL && !evaluate(feature, layer, filter, exclude_attributes, nz)) {
continue; continue;
} }
@@ -922,6 +934,7 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
long long tilesize = 1LL << (32 - oz); // source tile size in world coordinates long long tilesize = 1LL << (32 - oz); // source tile size in world coordinates
draw ring_closure(0, 0, 0); draw ring_closure(0, 0, 0);
bool sametile = (nz == oz && nx == ox && ny == oy && outlayer.extent >= layer.extent);
for (auto const &g : feature.geometry) { for (auto const &g : feature.geometry) {
if (g.op == mvt_closepath) { if (g.op == mvt_closepath) {
@@ -947,52 +960,59 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
g.y -= ny * outtilesize; g.y -= ny * outtilesize;
} }
// Clip to output tile if (!sametile) {
// Clip to output tile
long long xmin = LLONG_MAX; long long xmin = LLONG_MAX;
long long ymin = LLONG_MAX; long long ymin = LLONG_MAX;
long long xmax = LLONG_MIN; long long xmax = LLONG_MIN;
long long ymax = LLONG_MIN; long long ymax = LLONG_MIN;
for (auto const &g : geom) { for (auto const &g : geom) {
xmin = std::min(xmin, g.x); xmin = std::min(xmin, g.x);
ymin = std::min(ymin, g.y); ymin = std::min(ymin, g.y);
xmax = std::max(xmax, g.x); xmax = std::max(xmax, g.x);
ymax = std::max(ymax, g.y); ymax = std::max(ymax, g.y);
} }
long long b = outtilesize * buffer / 256; long long b = outtilesize * buffer / 256;
if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) { if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) {
continue; continue;
} }
if (t == VT_LINE) { if (t == VT_LINE) {
geom = clip_lines(geom, nz, buffer); geom = clip_lines(geom, nz, buffer);
} else if (t == VT_POLYGON) { } else if (t == VT_POLYGON) {
drawvec dv; drawvec dv;
geom = simple_clip_poly(geom, nz, buffer, dv, false); geom = simple_clip_poly(geom, nz, buffer, dv, false);
} else if (t == VT_POINT) { } else if (t == VT_POINT) {
geom = clip_point(geom, nz, buffer); geom = clip_point(geom, nz, buffer);
}
} }
// Scale to output tile extent // Scale to output tile extent
to_tile_scale(geom, nz, det); to_tile_scale(geom, nz, det);
// Clean geometries if (!sametile) {
// Clean geometries
geom = remove_noop(geom, t, 0);
if (t == VT_POLYGON) {
geom = clean_or_clip_poly(geom, 0, 0, false, false);
}
}
geom = remove_noop(geom, t, 0);
if (t == VT_POLYGON) { if (t == VT_POLYGON) {
geom = clean_or_clip_poly(geom, 0, 0, false, false);
geom = close_poly(geom); geom = close_poly(geom);
} }
tile_feature tf; tile_feature tf;
tf.geom = geom; tf.geom = std::move(geom);
tf.t = t; tf.t = t;
tf.has_id = feature.has_id; tf.has_id = feature.has_id;
tf.id = feature.id; tf.id = feature.id;
tf.tags = feature.tags; tf.tags = std::move(feature.tags);
tf.layer = &layer; tf.layer = &layer;
tf.seq = feature.seq; tf.seq = feature.seq;
@@ -1009,7 +1029,7 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
} }
if (outlayer.features.size() > 0) { if (outlayer.features.size() > 0) {
outtile.layers.push_back(outlayer); outtile.layers.push_back(std::move(outlayer));
} }
} }
decode.cpp
+1 -1
View File
@@ -277,7 +277,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
struct stat st; struct stat st;
std::vector<zxy> tiles; std::vector<zxy> tiles;
char *pmtiles_map; char *pmtiles_map = NULL;
std::vector<pmtiles::entry_zxy> entries; std::vector<pmtiles::entry_zxy> entries;
bool is_pmtiles = false; bool is_pmtiles = false;
evaluator.cpp
+159 -103
View File
@@ -1,37 +1,40 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <map> #include <unordered_map>
#include <functional>
#include "mvt.hpp" #include "mvt.hpp"
#include "evaluator.hpp" #include "evaluator.hpp"
#include "errors.hpp" #include "errors.hpp"
#include "milo/dtoa_milo.h" #include "milo/dtoa_milo.h"
static std::string mvt_value_to_string(mvt_value one, bool &fail) { static std::string mvt_value_to_string(mvt_value const &one, bool &fail) {
if (one.type == mvt_string) { switch (one.type) {
return one.string_value; case mvt_string:
} else if (one.type == mvt_float) { return one.get_string_value();
case mvt_float:
return milo::dtoa_milo(one.numeric_value.float_value); return milo::dtoa_milo(one.numeric_value.float_value);
} else if (one.type == mvt_double) { case mvt_double:
return milo::dtoa_milo(one.numeric_value.double_value); return milo::dtoa_milo(one.numeric_value.double_value);
} else if (one.type == mvt_int) { case mvt_int:
return std::to_string(one.numeric_value.int_value); return std::to_string(one.numeric_value.int_value);
} else if (one.type == mvt_uint) { case mvt_uint:
return std::to_string(one.numeric_value.uint_value); return std::to_string(one.numeric_value.uint_value);
} else if (one.type == mvt_sint) { case mvt_sint:
return std::to_string(one.numeric_value.sint_value); return std::to_string(one.numeric_value.sint_value);
} else if (one.type == mvt_bool) { case mvt_bool:
return one.numeric_value.bool_value ? "true" : "false"; return one.numeric_value.bool_value ? "true" : "false";
} else if (one.type == mvt_null) { case mvt_null:
fail = true; // null op string => null fail = true; // null op string => null
return ""; return "";
} else { case mvt_no_such_key:
default:
fprintf(stderr, "unhandled mvt_type %d\n", one.type); fprintf(stderr, "unhandled mvt_type %d\n", one.type);
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
} }
int compare_fsl(mvt_value one, json_object *two, bool &fail) { int compare_fsl(mvt_value const &one, json_object *two, bool &fail) {
// In FSL expressions, the attribute value is coerced to the type // In FSL expressions, the attribute value is coerced to the type
// of the JSON literal value it is being compared to. // of the JSON literal value it is being compared to.
// //
@@ -47,30 +50,38 @@ int compare_fsl(mvt_value one, json_object *two, bool &fail) {
if (two->type == JSON_NUMBER) { if (two->type == JSON_NUMBER) {
double lhs; double lhs;
if (one.type == mvt_string) { switch (one.type) {
case mvt_string: {
char *endptr = NULL; char *endptr = NULL;
const char *s = one.string_value.c_str(); const char *s = one.c_str();
lhs = strtod(s, &endptr); lhs = strtod(s, &endptr);
if (endptr == s) { if (endptr == s) {
fail = true; // non-numeric-string op number => null fail = true; // non-numeric-string op number => null
return 0; return 0;
} }
} else if (one.type == mvt_float) { } break;
case mvt_float:
lhs = one.numeric_value.float_value; lhs = one.numeric_value.float_value;
} else if (one.type == mvt_double) { break;
case mvt_double:
lhs = one.numeric_value.double_value; lhs = one.numeric_value.double_value;
} else if (one.type == mvt_int) { break;
case mvt_int:
lhs = one.numeric_value.int_value; lhs = one.numeric_value.int_value;
} else if (one.type == mvt_uint) { break;
case mvt_uint:
lhs = one.numeric_value.uint_value; lhs = one.numeric_value.uint_value;
} else if (one.type == mvt_sint) { break;
case mvt_sint:
lhs = one.numeric_value.sint_value; lhs = one.numeric_value.sint_value;
} else if (one.type == mvt_bool) { break;
case mvt_bool:
lhs = one.numeric_value.bool_value; lhs = one.numeric_value.bool_value;
} else if (one.type == mvt_null) { break;
case mvt_null:
fail = true; // null op number => null fail = true; // null op number => null
return 0; return 0;
} else { default:
fprintf(stderr, "unhandled mvt_type %d\n", one.type); fprintf(stderr, "unhandled mvt_type %d\n", one.type);
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
@@ -93,24 +104,34 @@ int compare_fsl(mvt_value one, json_object *two, bool &fail) {
if (two->type == JSON_TRUE || two->type == JSON_FALSE) { if (two->type == JSON_TRUE || two->type == JSON_FALSE) {
bool lhs; bool lhs;
if (one.type == mvt_string) { switch (one.type) {
lhs = one.string_value.size() > 0; case mvt_string:
} else if (one.type == mvt_float) { lhs = one.get_string_view().size() > 0;
break;
case mvt_float:
lhs = one.numeric_value.float_value != 0; lhs = one.numeric_value.float_value != 0;
} else if (one.type == mvt_double) { break;
case mvt_double:
lhs = one.numeric_value.double_value != 0; lhs = one.numeric_value.double_value != 0;
} else if (one.type == mvt_int) { break;
case mvt_int:
lhs = one.numeric_value.int_value != 0; lhs = one.numeric_value.int_value != 0;
} else if (one.type == mvt_uint) { break;
case mvt_uint:
lhs = one.numeric_value.uint_value != 0; lhs = one.numeric_value.uint_value != 0;
} else if (one.type == mvt_sint) { break;
case mvt_sint:
lhs = one.numeric_value.sint_value != 0; lhs = one.numeric_value.sint_value != 0;
} else if (one.type == mvt_bool) { break;
case mvt_bool:
lhs = one.numeric_value.bool_value; lhs = one.numeric_value.bool_value;
} else if (one.type == mvt_null) { break;
case mvt_null:
fail = true; // null op bool => null fail = true; // null op bool => null
return 0; return 0;
} else {
case mvt_no_such_key:
default:
fprintf(stderr, "unhandled mvt_type %d\n", one.type); fprintf(stderr, "unhandled mvt_type %d\n", one.type);
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
@@ -123,34 +144,45 @@ int compare_fsl(mvt_value one, json_object *two, bool &fail) {
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
int compare(mvt_value one, json_object *two, bool &fail) { int compare(mvt_value const &one, json_object *two, bool &fail) {
if (one.type == mvt_string) { switch (one.type) {
case mvt_string:
if (two->type != JSON_STRING) { if (two->type != JSON_STRING) {
fail = true; fail = true;
return false; // string vs non-string return false; // string vs non-string
} }
return strcmp(one.string_value.c_str(), two->value.string.string); return strcmp(one.c_str(), two->value.string.string);
}
if (one.type == mvt_double || one.type == mvt_float || one.type == mvt_int || one.type == mvt_uint || one.type == mvt_sint) { case mvt_double:
case mvt_float:
case mvt_int:
case mvt_uint:
case mvt_sint:
if (two->type != JSON_NUMBER) { if (two->type != JSON_NUMBER) {
fail = true; fail = true;
return false; // number vs non-number return false; // number vs non-number
} }
double v; double v;
if (one.type == mvt_double) { switch (one.type) {
case mvt_double:
v = one.numeric_value.double_value; v = one.numeric_value.double_value;
} else if (one.type == mvt_float) { break;
case mvt_float:
v = one.numeric_value.float_value; v = one.numeric_value.float_value;
} else if (one.type == mvt_int) { break;
case mvt_int:
v = one.numeric_value.int_value; v = one.numeric_value.int_value;
} else if (one.type == mvt_uint) { break;
case mvt_uint:
v = one.numeric_value.uint_value; v = one.numeric_value.uint_value;
} else if (one.type == mvt_sint) { break;
case mvt_sint:
v = one.numeric_value.sint_value; v = one.numeric_value.sint_value;
} else { break;
case mvt_no_such_key:
default:
fprintf(stderr, "Internal error: bad mvt type %d\n", one.type); fprintf(stderr, "Internal error: bad mvt type %d\n", one.type);
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
@@ -162,25 +194,29 @@ int compare(mvt_value one, json_object *two, bool &fail) {
} else { } else {
return 0; return 0;
} }
}
if (one.type == mvt_bool) { case mvt_bool:
if (two->type != JSON_TRUE && two->type != JSON_FALSE) { if (two->type != JSON_TRUE && two->type != JSON_FALSE) {
fail = true; fail = true;
return false; // bool vs non-bool return false; // bool vs non-bool
} }
bool b = two->type != JSON_FALSE; {
return one.numeric_value.bool_value > b; bool b = two->type != JSON_FALSE;
} return one.numeric_value.bool_value > b;
}
if (one.type == mvt_null) { case mvt_null:
if (two->type != JSON_NULL) { if (two->type != JSON_NULL) {
fail = true; fail = true;
return false; // null vs non-null return false; // null vs non-null
} }
return 0; // null equals null return 0; // null equals null
case mvt_no_such_key:
default:
break;
} }
fprintf(stderr, "Internal error: bad mvt type %d\n", one.type); fprintf(stderr, "Internal error: bad mvt type %d\n", one.type);
@@ -190,7 +226,7 @@ int compare(mvt_value one, json_object *two, bool &fail) {
// 0: false // 0: false
// 1: true // 1: true
// -1: incomparable (sql null), treated as false in final output // -1: incomparable (sql null), treated as false in final output
static int eval(std::map<std::string, mvt_value> const &feature, json_object *f, std::set<std::string> &exclude_attributes) { static int eval(std::function<mvt_value(std::string const &)> feature, json_object *f, std::set<std::string> &exclude_attributes) {
if (f != NULL) { if (f != NULL) {
if (f->type == JSON_TRUE) { if (f->type == JSON_TRUE) {
return 1; return 1;
@@ -236,10 +272,10 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
if (f->value.array.array[0]->type == JSON_STRING) { if (f->value.array.array[0]->type == JSON_STRING) {
// if LHS of a boolean conjunction is a string, it is an attribute reference // if LHS of a boolean conjunction is a string, it is an attribute reference
auto ff = feature.find(std::string(f->value.array.array[0]->value.string.string)); mvt_value ff = feature(std::string(f->value.array.array[0]->value.string.string));
if (ff != feature.end()) { if (ff.type != mvt_no_such_key) {
if (ff->second.type == mvt_bool) { if (ff.type == mvt_bool) {
lhs = ff->second.numeric_value.bool_value; lhs = ff.numeric_value.bool_value;
} else { } else {
lhs = -1; // not boolean: null lhs = -1; // not boolean: null
} }
@@ -280,9 +316,9 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
false)) { false)) {
mvt_value lhs; mvt_value lhs;
lhs.type = mvt_null; // attributes that aren't found are nulls lhs.type = mvt_null; // attributes that aren't found are nulls
auto ff = feature.find(std::string(f->value.array.array[0]->value.string.string)); mvt_value ff = feature(std::string(f->value.array.array[0]->value.string.string));
if (ff != feature.end()) { if (ff.type != mvt_no_such_key) {
lhs = ff->second; lhs = ff;
} }
if (f->value.array.array[2]->type == JSON_NULL && strcmp(f->value.array.array[1]->value.string.string, "is") == 0) { if (f->value.array.array[2]->type == JSON_NULL && strcmp(f->value.array.array[1]->value.string.string, "is") == 0) {
@@ -341,9 +377,8 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
} }
} }
int cmp = compare_fsl(ff->second, f->value.array.array[2], fail); int cmp = compare_fsl(ff, f->value.array.array[2], fail);
if (fail) { if (fail) {
printf("cast fail\n");
return -1; // null return -1; // null
} }
@@ -387,7 +422,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f)); fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f));
exit(EXIT_FILTER); exit(EXIT_FILTER);
} }
return feature.count(std::string(f->value.array.array[1]->value.string.string)) != 0; return feature(std::string(f->value.array.array[1]->value.string.string)).type != mvt_no_such_key;
} }
if (strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) { if (strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) {
@@ -395,7 +430,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f)); fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f));
exit(EXIT_FILTER); exit(EXIT_FILTER);
} }
return feature.count(std::string(f->value.array.array[1]->value.string.string)) == 0; return feature(std::string(f->value.array.array[1]->value.string.string)).type == mvt_no_such_key;
} }
} }
@@ -414,8 +449,8 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
exit(EXIT_FILTER); exit(EXIT_FILTER);
} }
auto ff = feature.find(std::string(f->value.array.array[1]->value.string.string)); mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string));
if (ff == feature.end()) { if (ff.type == mvt_no_such_key) {
static bool warned = false; static bool warned = false;
if (!warned) { if (!warned) {
const char *s = json_stringify(f); const char *s = json_stringify(f);
@@ -430,7 +465,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
} }
bool fail = false; bool fail = false;
int cmp = compare(ff->second, f->value.array.array[2], fail); int cmp = compare(ff, f->value.array.array[2], fail);
if (fail) { if (fail) {
static bool warned = false; static bool warned = false;
@@ -517,8 +552,8 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
exit(EXIT_FILTER); exit(EXIT_FILTER);
} }
auto ff = feature.find(std::string(f->value.array.array[1]->value.string.string)); mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string));
if (ff == feature.end()) { if (ff.type == mvt_no_such_key) {
static bool warned = false; static bool warned = false;
if (!warned) { if (!warned) {
const char *s = json_stringify(f); const char *s = json_stringify(f);
@@ -535,7 +570,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
bool found = false; bool found = false;
for (size_t i = 2; i < f->value.array.length; i++) { for (size_t i = 2; i < f->value.array.length; i++) {
bool fail = false; bool fail = false;
int cmp = compare(ff->second, f->value.array.array[i], fail); int cmp = compare(ff, f->value.array.array[i], fail);
if (fail) { if (fail) {
static bool warned = false; static bool warned = false;
@@ -584,7 +619,7 @@ static int eval(std::map<std::string, mvt_value> const &feature, json_object *f,
exit(EXIT_FILTER); exit(EXIT_FILTER);
} }
bool evaluate(std::map<std::string, mvt_value> const &feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes) { bool evaluate(std::function<mvt_value(std::string const &)> feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes) {
if (filter == NULL || filter->type != JSON_HASH) { if (filter == NULL || filter->type != JSON_HASH) {
fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter)); fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter));
exit(EXIT_JSON); exit(EXIT_JSON);
@@ -638,48 +673,69 @@ json_object *parse_filter(const char *s) {
return filter; return filter;
} }
bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set<std::string> &exclude_attributes, int z) { bool evaluate(std::unordered_map<std::string, mvt_value> const &feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes) {
if (filter != NULL) { std::function<mvt_value(std::string const &)> getter = [&](std::string const &key) {
std::map<std::string, mvt_value> attributes; auto f = feature.find(key);
if (f != feature.end()) {
for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { return f->second;
std::string key = layer.keys[feat.tags[t]]; } else {
const mvt_value &val = layer.values[feat.tags[t + 1]]; mvt_value v;
v.type = mvt_no_such_key;
attributes.insert(std::pair<std::string, mvt_value>(key, val)); v.numeric_value.null_value = 0;
return v;
} }
};
if (feat.has_id) { return evaluate(getter, layer, filter, exclude_attributes);
}
bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set<std::string> &exclude_attributes, int z) {
std::function<mvt_value(std::string const &)> getter = [&](std::string const &key) {
const static std::string dollar_id = "$id";
if (key == dollar_id && feat.has_id) {
mvt_value v; mvt_value v;
v.type = mvt_uint; v.type = mvt_uint;
v.numeric_value.uint_value = feat.id; v.numeric_value.uint_value = feat.id;
return v;
}
attributes.insert(std::pair<std::string, mvt_value>("$id", v)); const static std::string dollar_type = "$type";
if (key == dollar_type) {
mvt_value v;
v.type = mvt_string;
if (feat.type == mvt_point) {
const static std::string point = "Point";
v.set_string_value(point);
} else if (feat.type == mvt_linestring) {
const static std::string linestring = "LineString";
v.set_string_value(linestring);
} else if (feat.type == mvt_polygon) {
const static std::string polygon = "Polygon";
v.set_string_value(polygon);
}
return v;
}
const static std::string dollar_zoom = "$zoom";
if (key == dollar_zoom) {
mvt_value v2;
v2.type = mvt_uint;
v2.numeric_value.uint_value = z;
return v2;
}
for (size_t i = 0; i + 1 < feat.tags.size(); i += 2) {
if (layer.keys[feat.tags[i]] == key) {
return layer.values[feat.tags[i + 1]];
}
} }
mvt_value v; mvt_value v;
v.type = mvt_string; v.type = mvt_no_such_key;
v.numeric_value.null_value = 0;
return v;
};
if (feat.type == mvt_point) { return evaluate(getter, layer.name, filter, exclude_attributes);
v.string_value = "Point";
} else if (feat.type == mvt_linestring) {
v.string_value = "LineString";
} else if (feat.type == mvt_polygon) {
v.string_value = "Polygon";
}
attributes.insert(std::pair<std::string, mvt_value>("$type", v));
mvt_value v2;
v2.type = mvt_uint;
v2.numeric_value.uint_value = z;
attributes.insert(std::pair<std::string, mvt_value>("$zoom", v2));
if (!evaluate(attributes, layer.name, filter, exclude_attributes)) {
return false;
}
}
return true;
} }
evaluator.hpp
+2 -2
View File
@@ -1,13 +1,13 @@
#ifndef EVALUATOR_HPP #ifndef EVALUATOR_HPP
#define EVALUATOR HPP #define EVALUATOR HPP
#include <map> #include <unordered_map>
#include <string> #include <string>
#include <set> #include <set>
#include "jsonpull/jsonpull.h" #include "jsonpull/jsonpull.h"
#include "mvt.hpp" #include "mvt.hpp"
bool evaluate(std::map<std::string, mvt_value> const &feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes); bool evaluate(std::unordered_map<std::string, mvt_value> const &feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes);
json_object *parse_filter(const char *s); json_object *parse_filter(const char *s);
json_object *read_filter(const char *fname); json_object *read_filter(const char *fname);
geometry.hpp
+4 -4
View File
@@ -100,16 +100,16 @@ void visvalingam(drawvec &ls, size_t start, size_t end, double threshold, size_t
int pnpoly(const drawvec &vert, size_t start, size_t nvert, long long testx, long long testy); int pnpoly(const drawvec &vert, size_t start, size_t nvert, long long testx, long long testy);
double distance_from_line(long long point_x, long long point_y, long long segA_x, long long segA_y, long long segB_x, long long segB_y); double distance_from_line(long long point_x, long long point_y, long long segA_x, long long segA_y, long long segB_x, long long segB_y);
std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny, std::string overzoom(const mvt_tile &tile, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress, int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles, std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
bool demultiply, json_object *filter, bool preserve_input_order, bool demultiply, json_object *filter, bool preserve_input_order,
std::map<std::string, attribute_op> const &attribute_accum); std::unordered_map<std::string, attribute_op> const &attribute_accum);
std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny, std::string overzoom(const std::string &s, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress, int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles, std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
bool demultiply, json_object *filter, bool preserve_input_order, bool demultiply, json_object *filter, bool preserve_input_order,
std::map<std::string, attribute_op> const &attribute_accum); std::unordered_map<std::string, attribute_op> const &attribute_accum);
#endif #endif
main.cpp
+7 -7
View File
@@ -448,7 +448,7 @@ void *run_sort(void *v) {
return NULL; return NULL;
} }
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector<struct reader> *readers, std::atomic<long long> *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, double *area_sum, bool want_dist, bool filters) { void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, std::vector<struct reader> *readers, std::atomic<long long> *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > *layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::unordered_map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, double *area_sum, bool want_dist, bool filters) {
long long segs[CPUS + 1]; long long segs[CPUS + 1];
segs[0] = 0; segs[0] = 0;
segs[CPUS] = len; segs[CPUS] = len;
@@ -649,7 +649,7 @@ struct read_parallel_arg {
unsigned *initial_y = NULL; unsigned *initial_y = NULL;
std::string layername = ""; std::string layername = "";
bool uses_gamma = false; bool uses_gamma = false;
std::map<std::string, int> const *attribute_types = NULL; std::unordered_map<std::string, int> const *attribute_types = NULL;
double *dist_sum = NULL; double *dist_sum = NULL;
size_t *dist_count = NULL; size_t *dist_count = NULL;
double *area_sum = NULL; double *area_sum = NULL;
@@ -693,7 +693,7 @@ void *run_read_parallel(void *v) {
return NULL; return NULL;
} }
void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::atomic<int> *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector<struct reader> *readers, std::atomic<long long> *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, double *area_sum, bool want_dist, bool filters) { void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::atomic<int> *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, std::vector<struct reader> *readers, std::atomic<long long> *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, int basezoom, int source, std::vector<std::map<std::string, layermap_entry> > &layermaps, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::unordered_map<std::string, int> const *attribute_types, int separator, double *dist_sum, size_t *dist_count, double *area_sum, bool want_dist, bool filters) {
// This has to kick off an intermediate thread to start the parser threads, // This has to kick off an intermediate thread to start the parser threads,
// so the main thread can get back to reading the next input stage while // so the main thread can get back to reading the next input stage while
// the intermediate thread waits for the completion of the parser threads. // the intermediate thread waits for the completion of the parser threads.
@@ -1237,7 +1237,7 @@ int vertexcmp(const void *void1, const void *void2) {
return 0; return 0;
} }
std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm, std::map<std::string, attribute_op> const *attribute_accum, std::map<std::string, std::string> const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) { std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::unordered_map<std::string, int> const *attribute_types, const char *pgm, std::unordered_map<std::string, attribute_op> const *attribute_accum, std::map<std::string, std::string> const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) {
int ret = EXIT_SUCCESS; int ret = EXIT_SUCCESS;
std::vector<struct reader> readers; std::vector<struct reader> readers;
@@ -2845,7 +2845,7 @@ static bool has_name(struct option *long_options, int *pl) {
return false; return false;
} }
void set_attribute_type(std::map<std::string, int> &attribute_types, const char *arg) { void set_attribute_type(std::unordered_map<std::string, int> &attribute_types, const char *arg) {
const char *s = strchr(arg, ':'); const char *s = strchr(arg, ':');
if (s == NULL) { if (s == NULL) {
fprintf(stderr, "-T%s option must be in the form -Tname:type\n", arg); fprintf(stderr, "-T%s option must be in the form -Tname:type\n", arg);
@@ -3005,8 +3005,8 @@ int main(int argc, char **argv) {
bool guess_cluster_maxzoom = false; bool guess_cluster_maxzoom = false;
std::set<std::string> exclude, include; std::set<std::string> exclude, include;
std::map<std::string, int> attribute_types; std::unordered_map<std::string, int> attribute_types;
std::map<std::string, attribute_op> attribute_accum; std::unordered_map<std::string, attribute_op> attribute_accum;
std::map<std::string, std::string> attribute_descriptions; std::map<std::string, std::string> attribute_descriptions;
int exclude_all = 0; int exclude_all = 0;
int read_parallel = 0; int read_parallel = 0;
mvt.cpp
+183 -66
View File
@@ -110,7 +110,7 @@ int compress(std::string const &input, std::string &output, bool gz) {
return 0; return 0;
} }
bool mvt_tile::decode(std::string &message, bool &was_compressed) { bool mvt_tile::decode(const std::string &message, bool &was_compressed) {
layers.clear(); layers.clear();
std::string src; std::string src;
@@ -127,6 +127,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
} }
protozero::pbf_reader reader(src); protozero::pbf_reader reader(src);
std::shared_ptr<std::string> string_pool = std::make_shared<std::string>();
while (reader.next()) { while (reader.next()) {
switch (reader.tag()) { switch (reader.tag()) {
@@ -157,7 +158,12 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
switch (value_reader.tag()) { switch (value_reader.tag()) {
case 1: /* string */ case 1: /* string */
value.type = mvt_string; value.type = mvt_string;
value.string_value = value_reader.get_string(); value.s = string_pool;
{
auto v = value_reader.get_view();
std::string_view sv(v.data(), v.size());
value.set_string_value(sv);
}
break; break;
case 2: /* float */ case 2: /* float */
@@ -196,7 +202,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
} }
} }
layer.values.push_back(value); layer.values.push_back(std::move(value));
break; break;
} }
@@ -224,6 +230,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
case 2: /* tag */ case 2: /* tag */
{ {
auto pi = feature_reader.get_packed_uint32(); auto pi = feature_reader.get_packed_uint32();
feature.tags.reserve(std::distance(pi.first, pi.second));
for (auto it = pi.first; it != pi.second; ++it) { for (auto it = pi.first; it != pi.second; ++it) {
feature.tags.push_back(*it); feature.tags.push_back(*it);
} }
@@ -237,6 +244,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
case 4: /* geometry */ case 4: /* geometry */
{ {
auto pi = feature_reader.get_packed_uint32(); auto pi = feature_reader.get_packed_uint32();
geoms.reserve(std::distance(pi.first, pi.second));
for (auto it = pi.first; it != pi.second; ++it) { for (auto it = pi.first; it != pi.second; ++it) {
geoms.push_back(*it); geoms.push_back(*it);
} }
@@ -250,6 +258,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
} }
long long px = 0, py = 0; long long px = 0, py = 0;
feature.geometry.reserve(geoms.size()); // probably not quite right, but still plausible
for (size_t g = 0; g < geoms.size(); g++) { for (size_t g = 0; g < geoms.size(); g++) {
uint32_t geom = geoms[g]; uint32_t geom = geoms[g];
uint32_t op = geom & 7; uint32_t op = geom & 7;
@@ -261,14 +270,14 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
py += protozero::decode_zigzag32(geoms[g + 2]); py += protozero::decode_zigzag32(geoms[g + 2]);
g += 2; g += 2;
feature.geometry.push_back(mvt_geometry(op, px, py)); feature.geometry.emplace_back(op, px, py);
} }
} else { } else {
feature.geometry.push_back(mvt_geometry(op, 0, 0)); feature.geometry.emplace_back(op, 0, 0);
} }
} }
layer.features.push_back(feature); layer.features.push_back(std::move(feature));
break; break;
} }
@@ -278,14 +287,7 @@ bool mvt_tile::decode(std::string &message, bool &was_compressed) {
} }
} }
for (size_t i = 0; i < layer.keys.size(); i++) { layers.push_back(std::move(layer));
layer.key_map.insert(std::pair<std::string, size_t>(layer.keys[i], i));
}
for (size_t i = 0; i < layer.values.size(); i++) {
layer.value_map.insert(std::pair<mvt_value, size_t>(layer.values[i], i));
}
layers.push_back(layer);
break; break;
} }
@@ -313,7 +315,6 @@ struct sorted_value {
std::string mvt_tile::encode() { std::string mvt_tile::encode() {
std::string data; std::string data;
protozero::pbf_writer writer(data); protozero::pbf_writer writer(data);
for (size_t i = 0; i < layers.size(); i++) { for (size_t i = 0; i < layers.size(); i++) {
@@ -335,32 +336,40 @@ std::string mvt_tile::encode() {
protozero::pbf_writer value_writer(value_string); protozero::pbf_writer value_writer(value_string);
mvt_value &pbv = layers[i].values[v]; mvt_value &pbv = layers[i].values[v];
if (pbv.type == mvt_string) { switch (pbv.type) {
value_writer.add_string(1, pbv.string_value); case mvt_string:
} else if (pbv.type == mvt_float) { value_writer.add_string(1, pbv.get_string_value());
break;
case mvt_float:
value_writer.add_float(2, pbv.numeric_value.float_value); value_writer.add_float(2, pbv.numeric_value.float_value);
} else if (pbv.type == mvt_double) { break;
case mvt_double:
value_writer.add_double(3, pbv.numeric_value.double_value); value_writer.add_double(3, pbv.numeric_value.double_value);
} else if (pbv.type == mvt_int) { break;
case mvt_int:
value_writer.add_int64(4, pbv.numeric_value.int_value); value_writer.add_int64(4, pbv.numeric_value.int_value);
} else if (pbv.type == mvt_uint) { break;
case mvt_uint:
value_writer.add_uint64(5, pbv.numeric_value.uint_value); value_writer.add_uint64(5, pbv.numeric_value.uint_value);
} else if (pbv.type == mvt_sint) { break;
case mvt_sint:
value_writer.add_sint64(6, pbv.numeric_value.sint_value); value_writer.add_sint64(6, pbv.numeric_value.sint_value);
} else if (pbv.type == mvt_bool) { break;
case mvt_bool:
value_writer.add_bool(7, pbv.numeric_value.bool_value); value_writer.add_bool(7, pbv.numeric_value.bool_value);
} else if (pbv.type == mvt_null) { break;
case mvt_null:
fprintf(stderr, "Internal error: trying to write null attribute to tile\n"); fprintf(stderr, "Internal error: trying to write null attribute to tile\n");
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} else { default:
fprintf(stderr, "Internal error: trying to write undefined attribute type to tile\n"); fprintf(stderr, "Internal error: trying to write undefined attribute type to tile\n");
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
sorted_value sv; sorted_value sv;
sv.val = value_string; sv.val = std::move(value_string);
sv.orig = v; sv.orig = v;
sorted_values.push_back(sv); sorted_values.push_back(std::move(sv));
} }
std::sort(sorted_values.begin(), sorted_values.end()); std::sort(sorted_values.begin(), sorted_values.end());
@@ -456,15 +465,70 @@ bool mvt_value::operator<(const mvt_value &o) const {
return true; return true;
} }
if (type == o.type) { if (type == o.type) {
if ((type == mvt_string && string_value < o.string_value) || switch (type) {
(type == mvt_float && numeric_value.float_value < o.numeric_value.float_value) || case mvt_string:
(type == mvt_double && numeric_value.double_value < o.numeric_value.double_value) || return get_string_view() < o.get_string_view();
(type == mvt_int && numeric_value.int_value < o.numeric_value.int_value) ||
(type == mvt_uint && numeric_value.uint_value < o.numeric_value.uint_value) || case mvt_float:
(type == mvt_sint && numeric_value.sint_value < o.numeric_value.sint_value) || return numeric_value.float_value < o.numeric_value.float_value;
(type == mvt_bool && numeric_value.bool_value < o.numeric_value.bool_value) ||
(type == mvt_null && numeric_value.null_value < o.numeric_value.null_value)) { case mvt_double:
return true; return numeric_value.double_value < o.numeric_value.double_value;
case mvt_int:
return numeric_value.int_value < o.numeric_value.int_value;
case mvt_uint:
return numeric_value.uint_value < o.numeric_value.uint_value;
case mvt_sint:
return numeric_value.sint_value < o.numeric_value.sint_value;
case mvt_bool:
return numeric_value.bool_value < o.numeric_value.bool_value;
case mvt_null:
return numeric_value.null_value < o.numeric_value.null_value;
default:
fprintf(stderr, "mvt_value::operator<<: can't happen\n");
exit(EXIT_IMPOSSIBLE);
}
}
return false;
}
bool mvt_value::operator==(const mvt_value &o) const {
if (type == o.type) {
switch (type) {
case mvt_string:
return get_string_view() == o.get_string_view();
case mvt_float:
return numeric_value.float_value == o.numeric_value.float_value;
case mvt_double:
return numeric_value.double_value == o.numeric_value.double_value;
case mvt_int:
return numeric_value.int_value == o.numeric_value.int_value;
case mvt_uint:
return numeric_value.uint_value == o.numeric_value.uint_value;
case mvt_sint:
return numeric_value.sint_value == o.numeric_value.sint_value;
case mvt_bool:
return numeric_value.bool_value == o.numeric_value.bool_value;
case mvt_null:
return numeric_value.null_value == o.numeric_value.null_value;
default:
fprintf(stderr, "mvt_value::operator==: can't happen\n");
exit(EXIT_IMPOSSIBLE);
} }
} }
@@ -493,47 +557,62 @@ static std::string quote(std::string const &s) {
} }
std::string mvt_value::toString() const { std::string mvt_value::toString() const {
if (type == mvt_string) { switch (type) {
return quote(string_value); case mvt_string:
} else if (type == mvt_int) { return quote(get_string_value());
case mvt_int:
return std::to_string(numeric_value.int_value); return std::to_string(numeric_value.int_value);
} else if (type == mvt_double) { case mvt_double: {
double v = numeric_value.double_value; double v = numeric_value.double_value;
if (v == (long long) v) { if (v == (long long) v) {
return std::to_string((long long) v); return std::to_string((long long) v);
} else { } else {
return milo::dtoa_milo(v); return milo::dtoa_milo(v);
} }
} else if (type == mvt_float) { }
case mvt_float: {
double v = numeric_value.float_value; double v = numeric_value.float_value;
if (v == (long long) v) { if (v == (long long) v) {
return std::to_string((long long) v); return std::to_string((long long) v);
} else { } else {
return milo::dtoa_milo(v); return milo::dtoa_milo(v);
} }
} else if (type == mvt_sint) { }
case mvt_sint:
return std::to_string(numeric_value.sint_value); return std::to_string(numeric_value.sint_value);
} else if (type == mvt_uint) { case mvt_uint:
return std::to_string(numeric_value.uint_value); return std::to_string(numeric_value.uint_value);
} else if (type == mvt_bool) { case mvt_bool:
return numeric_value.bool_value ? "true" : "false"; return numeric_value.bool_value ? "true" : "false";
} else if (type == mvt_null) { case mvt_null:
return "null"; return "null";
} else { default:
return "unknown"; return "unknown";
} }
} }
void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) { void mvt_layer::tag(mvt_feature &feature, std::string const &key, mvt_value const &value) {
size_t ko, vo; size_t ko, vo;
std::map<std::string, size_t>::iterator ki = key_map.find(key); // initialize lazily the first time anyone tags an attribute
std::map<mvt_value, size_t>::iterator vi = value_map.find(value); // to save the time of doing it in decode, which never actually matters.
// only tile writers actually need this.
if (key_map.size() == 0) {
for (size_t i = 0; i < keys.size(); i++) {
key_map.emplace(keys[i], i);
}
for (size_t i = 0; i < values.size(); i++) {
value_map.emplace(values[i], i);
}
}
std::unordered_map<std::string, size_t>::iterator ki = key_map.find(key);
std::unordered_map<mvt_value, size_t>::iterator vi = value_map.find(value);
if (ki == key_map.end()) { if (ki == key_map.end()) {
ko = keys.size(); ko = keys.size();
keys.push_back(key); keys.push_back(key);
key_map.insert(std::pair<std::string, size_t>(key, ko)); key_map.emplace(key, ko);
} else { } else {
ko = ki->second; ko = ki->second;
} }
@@ -541,7 +620,7 @@ void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) {
if (vi == value_map.end()) { if (vi == value_map.end()) {
vo = values.size(); vo = values.size();
values.push_back(value); values.push_back(value);
value_map.insert(std::pair<mvt_value, size_t>(value, vo)); value_map.emplace(value, vo);
} else { } else {
vo = vi->second; vo = vi->second;
} }
@@ -629,7 +708,8 @@ bool is_unsigned_integer(const char *s, unsigned long long *v) {
mvt_value stringified_to_mvt_value(int type, const char *s) { mvt_value stringified_to_mvt_value(int type, const char *s) {
mvt_value tv; mvt_value tv;
if (type == mvt_double) { switch (type) {
case mvt_double: {
long long v; long long v;
unsigned long long uv; unsigned long long uv;
if (is_unsigned_integer(s, &uv)) { if (is_unsigned_integer(s, &uv)) {
@@ -668,15 +748,18 @@ mvt_value stringified_to_mvt_value(int type, const char *s) {
} }
} }
} }
} else if (type == mvt_bool) { } break;
case mvt_bool:
tv.type = mvt_bool; tv.type = mvt_bool;
tv.numeric_value.bool_value = (s[0] == 't'); tv.numeric_value.bool_value = (s[0] == 't');
} else if (type == mvt_null) { break;
case mvt_null:
tv.type = mvt_null; tv.type = mvt_null;
tv.numeric_value.null_value = 0; tv.numeric_value.null_value = 0;
} else { break;
default:
tv.type = mvt_string; tv.type = mvt_string;
tv.string_value = s; tv.set_string_value(s);
} }
return tv; return tv;
@@ -690,34 +773,68 @@ mvt_value stringified_to_mvt_value(int type, const char *s) {
serial_val mvt_value_to_serial_val(mvt_value const &v) { serial_val mvt_value_to_serial_val(mvt_value const &v) {
serial_val sv; serial_val sv;
if (v.type == mvt_string) { switch (v.type) {
case mvt_string:
sv.type = mvt_string; sv.type = mvt_string;
sv.s = v.string_value; sv.s = v.get_string_value();
} else if (v.type == mvt_float) { break;
case mvt_float:
sv.type = mvt_double; sv.type = mvt_double;
sv.s = milo::dtoa_milo(v.numeric_value.float_value); sv.s = milo::dtoa_milo(v.numeric_value.float_value);
} else if (v.type == mvt_double) { break;
case mvt_double:
sv.type = mvt_double; sv.type = mvt_double;
sv.s = milo::dtoa_milo(v.numeric_value.double_value); sv.s = milo::dtoa_milo(v.numeric_value.double_value);
} else if (v.type == mvt_int) { break;
case mvt_int:
sv.type = mvt_double; sv.type = mvt_double;
sv.s = std::to_string(v.numeric_value.int_value); sv.s = std::to_string(v.numeric_value.int_value);
} else if (v.type == mvt_uint) { break;
case mvt_uint:
sv.type = mvt_double; sv.type = mvt_double;
sv.s = std::to_string(v.numeric_value.uint_value); sv.s = std::to_string(v.numeric_value.uint_value);
} else if (v.type == mvt_sint) { break;
case mvt_sint:
sv.type = mvt_double; sv.type = mvt_double;
sv.s = std::to_string(v.numeric_value.sint_value); sv.s = std::to_string(v.numeric_value.sint_value);
} else if (v.type == mvt_bool) { break;
case mvt_bool:
sv.type = mvt_bool; sv.type = mvt_bool;
sv.s = v.numeric_value.bool_value ? "true" : "false"; sv.s = v.numeric_value.bool_value ? "true" : "false";
} else if (v.type == mvt_null) { break;
case mvt_null:
sv.type = mvt_null; sv.type = mvt_null;
sv.s = "null"; sv.s = "null";
} else { break;
default:
fprintf(stderr, "unhandled mvt_type %d\n", v.type); fprintf(stderr, "unhandled mvt_type %d\n", v.type);
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
return sv; return sv;
} }
// This extracts an integer value from an mvt_value
long long mvt_value_to_long_long(mvt_value const &v) {
switch (v.type) {
case mvt_string:
return atoll(v.c_str());
case mvt_float:
return v.numeric_value.float_value;
case mvt_double:
return v.numeric_value.double_value;
case mvt_int:
return v.numeric_value.int_value;
case mvt_uint:
return v.numeric_value.uint_value;
case mvt_sint:
return v.numeric_value.sint_value;
case mvt_bool:
return v.numeric_value.bool_value;
case mvt_null:
return 0;
default:
fprintf(stderr, "unhandled mvt_type %d\n", v.type);
exit(EXIT_IMPOSSIBLE);
}
}
mvt.hpp
+79 -7
View File
@@ -3,9 +3,14 @@
#include <sqlite3.h> #include <sqlite3.h>
#include <string> #include <string>
#include <map> #include <string_view>
#include <unordered_map>
#include <set> #include <set>
#include <vector> #include <vector>
#include <optional>
#include <memory>
#include "errors.hpp"
struct mvt_value; struct mvt_value;
struct mvt_layer; struct mvt_layer;
@@ -66,11 +71,14 @@ enum mvt_value_type {
mvt_sint, mvt_sint,
mvt_bool, mvt_bool,
mvt_null, mvt_null,
mvt_no_such_key,
}; };
struct mvt_value { struct mvt_value {
mvt_value_type type; mvt_value_type type;
std::string string_value; std::shared_ptr<std::string> s;
union { union {
float float_value; float float_value;
double double_value; double double_value;
@@ -79,18 +87,81 @@ struct mvt_value {
long long sint_value; long long sint_value;
bool bool_value; bool bool_value;
int null_value; int null_value;
struct {
size_t off;
size_t len;
} string_value;
} numeric_value; } numeric_value;
std::string get_string_value() const {
return std::string(*s, numeric_value.string_value.off, numeric_value.string_value.len);
}
std::string_view get_string_view() const {
return std::string_view(s->c_str() + numeric_value.string_value.off, numeric_value.string_value.len);
}
const char *c_str() const {
return s->c_str() + numeric_value.string_value.off;
}
void set_string_value(const std::string_view &val) {
if (s == nullptr) {
s = std::make_shared<std::string>();
}
type = mvt_string;
numeric_value.string_value.off = s->size();
numeric_value.string_value.len = val.size();
s->append(val);
s->push_back('\0');
}
bool operator<(const mvt_value &o) const; bool operator<(const mvt_value &o) const;
bool operator==(const mvt_value &o) const;
std::string toString() const; std::string toString() const;
mvt_value() { mvt_value() {
this->type = mvt_double; this->type = mvt_double;
this->string_value = "";
this->numeric_value.double_value = 0; this->numeric_value.double_value = 0;
} }
}; };
template <>
struct std::hash<mvt_value> {
std::size_t operator()(const mvt_value &k) const {
switch (k.type) {
case mvt_string:
return std::hash<std::string_view>()(k.get_string_view());
case mvt_float:
return std::hash<float>()(k.numeric_value.float_value);
case mvt_double:
return std::hash<double>()(k.numeric_value.double_value);
case mvt_int:
return std::hash<long long>()(k.numeric_value.int_value);
case mvt_uint:
return std::hash<unsigned long long>()(k.numeric_value.uint_value);
case mvt_sint:
return std::hash<long long>()(k.numeric_value.sint_value);
case mvt_bool:
return std::hash<bool>()(k.numeric_value.bool_value);
case mvt_null:
return std::hash<int>()(k.numeric_value.null_value);
default:
fprintf(stderr, "mvt_value hash can't happen\n");
exit(EXIT_IMPOSSIBLE);
}
}
};
struct mvt_layer { struct mvt_layer {
int version = 0; int version = 0;
std::string name = ""; std::string name = "";
@@ -100,18 +171,18 @@ struct mvt_layer {
long long extent = 0; long long extent = 0;
// Add a key-value pair to a feature, using this layer's constant pool // Add a key-value pair to a feature, using this layer's constant pool
void tag(mvt_feature &feature, std::string key, mvt_value value); void tag(mvt_feature &feature, std::string const &key, mvt_value const &value);
// For tracking the key-value constants already used in this layer // For tracking the key-value constants already used in this layer
std::map<std::string, size_t> key_map{}; std::unordered_map<std::string, size_t> key_map{};
std::map<mvt_value, size_t> value_map{}; std::unordered_map<mvt_value, size_t> value_map{};
}; };
struct mvt_tile { struct mvt_tile {
std::vector<mvt_layer> layers{}; std::vector<mvt_layer> layers{};
std::string encode(); std::string encode();
bool decode(std::string &message, bool &was_compressed); bool decode(const std::string &message, bool &was_compressed);
}; };
bool is_compressed(std::string const &data); bool is_compressed(std::string const &data);
@@ -120,6 +191,7 @@ int compress(std::string const &input, std::string &output, bool gz);
int dezig(unsigned n); int dezig(unsigned n);
mvt_value stringified_to_mvt_value(int type, const char *s); mvt_value stringified_to_mvt_value(int type, const char *s);
long long mvt_value_to_long_long(mvt_value const &v);
bool is_integer(const char *s, long long *v); bool is_integer(const char *s, long long *v);
bool is_unsigned_integer(const char *s, unsigned long long *v); bool is_unsigned_integer(const char *s, unsigned long long *v);
overzoom.cpp
+1 -1
View File
@@ -17,7 +17,7 @@ int buffer = 5; // tippecanoe-style: mvt buffer == extent * buffer / 256;
bool demultiply = false; bool demultiply = false;
std::string filter; std::string filter;
bool preserve_input_order = false; bool preserve_input_order = false;
std::map<std::string, attribute_op> attribute_accum; std::unordered_map<std::string, attribute_op> attribute_accum;
std::set<std::string> keep; std::set<std::string> keep;
pmtiles/pmtiles.hpp
+3 -3
View File
@@ -166,11 +166,11 @@ struct entryv3 {
} }
}; };
struct { struct entryv3_cmp {
bool operator()(entryv3 a, entryv3 b) const { bool operator()(entryv3 a, entryv3 b) const {
return a.tile_id < b.tile_id; return a.tile_id < b.tile_id;
} }
} entryv3_cmp; };
struct entry_zxy { struct entry_zxy {
uint8_t z; uint8_t z;
@@ -388,7 +388,7 @@ inline uint64_t zxy_to_tileid(uint8_t z, uint32_t x, uint32_t y) {
if (z > 31) { if (z > 31) {
throw std::overflow_error("tile zoom exceeds 64-bit limit"); throw std::overflow_error("tile zoom exceeds 64-bit limit");
} }
if (x > (1 << z) - 1 || y > (1 << z) - 1) { if (x > (1U << z) - 1 || y > (1U << z) - 1) {
throw std::overflow_error("tile x/y outside zoom level bounds"); throw std::overflow_error("tile x/y outside zoom level bounds");
} }
uint64_t acc = 0; uint64_t acc = 0;
pmtiles_file.cpp
+1 -1
View File
@@ -263,7 +263,7 @@ void mbtiles_map_image_to_pmtiles(char *fname, metadata m, bool tile_compression
// finalize PMTiles archive. // finalize PMTiles archive.
{ {
std::sort(entries.begin(), entries.end(), pmtiles::entryv3_cmp); std::sort(entries.begin(), entries.end(), pmtiles::entryv3_cmp());
std::string root_bytes; std::string root_bytes;
std::string leaves_bytes; std::string leaves_bytes;
serial.cpp
+1 -1
View File
@@ -878,7 +878,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
return 1; return 1;
} }
void coerce_value(std::string const &key, int &vt, std::string &val, std::map<std::string, int> const *attribute_types) { void coerce_value(std::string const &key, int &vt, std::string &val, std::unordered_map<std::string, int> const *attribute_types) {
auto a = (*attribute_types).find(key); auto a = (*attribute_types).find(key);
if (a != attribute_types->end()) { if (a != attribute_types->end()) {
if (a->second == mvt_string) { if (a->second == mvt_string) {
serial.hpp
+2 -2
View File
@@ -175,7 +175,7 @@ struct serialization_state {
std::map<std::string, layermap_entry> *layermap = NULL; std::map<std::string, layermap_entry> *layermap = NULL;
std::map<std::string, int> const *attribute_types = NULL; std::unordered_map<std::string, int> const *attribute_types = NULL;
std::set<std::string> *exclude = NULL; std::set<std::string> *exclude = NULL;
std::set<std::string> *include = NULL; std::set<std::string> *include = NULL;
int exclude_all = 0; int exclude_all = 0;
@@ -228,6 +228,6 @@ struct node {
int nodecmp(const void *void1, const void *void2); int nodecmp(const void *void1, const void *void2);
int serialize_feature(struct serialization_state *sst, serial_feature &sf); int serialize_feature(struct serialization_state *sst, serial_feature &sf);
void coerce_value(std::string const &key, int &vt, std::string &val, std::map<std::string, int> const *attribute_types); void coerce_value(std::string const &key, int &vt, std::string &val, std::unordered_map<std::string, int> const *attribute_types);
#endif #endif
tile-join.cpp
+5 -5
View File
@@ -140,7 +140,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
mvt_feature feat = layer.features[f]; mvt_feature feat = layer.features[f];
std::set<std::string> exclude_attributes; std::set<std::string> exclude_attributes;
if (!evaluate(feat, layer, filter, exclude_attributes, z)) { if (filter != NULL && !evaluate(feat, layer, filter, exclude_attributes, z)) {
continue; continue;
} }
@@ -197,7 +197,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
mvt_value outval; mvt_value outval;
if (attr_type == mvt_string) { if (attr_type == mvt_string) {
outval.type = mvt_string; outval.type = mvt_string;
outval.string_value = joinval; outval.set_string_value(joinval);
} else { } else {
outval.type = mvt_double; outval.type = mvt_double;
outval.numeric_value.double_value = atof(joinval.c_str()); outval.numeric_value.double_value = atof(joinval.c_str());
@@ -688,8 +688,8 @@ struct tileset_reader {
perror("pthread_mutex_lock"); perror("pthread_mutex_lock");
} }
std::function<mvt_tile(zxy)> getter = [&](zxy tile) { std::function<mvt_tile(zxy)> getter = [&](zxy tileno) {
return get_tile(tile); return get_tile(tileno);
}; };
mvt_tile source = cache.get(parent_tile, getter); mvt_tile source = cache.get(parent_tile, getter);
@@ -699,7 +699,7 @@ struct tileset_reader {
} }
if (source.layers.size() != 0) { if (source.layers.size() != 0) {
std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles, false, NULL, false, std::map<std::string, attribute_op>()); std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles, false, NULL, false, std::unordered_map<std::string, attribute_op>());
return ret; return ret;
} }
tile.cpp
+14 -13
View File
@@ -8,6 +8,7 @@
#include <stack> #include <stack>
#include <vector> #include <vector>
#include <map> #include <map>
#include <unordered_map>
#include <set> #include <set>
#include <algorithm> #include <algorithm>
#include <stdio.h> #include <stdio.h>
@@ -216,7 +217,7 @@ void decode_meta(std::vector<long long> const &metakeys, std::vector<long long>
mvt_value key = retrieve_string(metakeys[i], stringpool, NULL); mvt_value key = retrieve_string(metakeys[i], stringpool, NULL);
mvt_value value = retrieve_string(metavals[i], stringpool, &otype); mvt_value value = retrieve_string(metavals[i], stringpool, &otype);
layer.tag(feature, key.string_value, value); layer.tag(feature, key.get_string_value(), value);
} }
} }
@@ -226,9 +227,9 @@ static int metacmp(const std::vector<long long> &keys1, const std::vector<long l
mvt_value key1 = retrieve_string(keys1[i], stringpool1, NULL); mvt_value key1 = retrieve_string(keys1[i], stringpool1, NULL);
mvt_value key2 = retrieve_string(keys2[i], stringpool2, NULL); mvt_value key2 = retrieve_string(keys2[i], stringpool2, NULL);
if (key1.string_value < key2.string_value) { if (key1.get_string_view() < key2.get_string_view()) {
return -1; return -1;
} else if (key1.string_value > key2.string_value) { } else if (key1.get_string_view() > key2.get_string_view()) {
return 1; return 1;
} }
@@ -258,7 +259,7 @@ static int metacmp(const std::vector<long long> &keys1, const std::vector<long l
} }
} }
static mvt_value find_attribute_value(const struct coalesce *c1, std::string key) { static mvt_value find_attribute_value(const struct coalesce *c1, std::string const &key) {
if (key == ORDER_BY_SIZE) { if (key == ORDER_BY_SIZE) {
mvt_value v; mvt_value v;
v.type = mvt_double; v.type = mvt_double;
@@ -272,7 +273,7 @@ static mvt_value find_attribute_value(const struct coalesce *c1, std::string key
for (size_t i = 0; i < keys1.size(); i++) { for (size_t i = 0; i < keys1.size(); i++) {
mvt_value key1 = retrieve_string(keys1[i], stringpool1, NULL); mvt_value key1 = retrieve_string(keys1[i], stringpool1, NULL);
if (key == key1.string_value) { if (key == key1.get_string_value()) {
return retrieve_string(values1[i], stringpool1, NULL); return retrieve_string(values1[i], stringpool1, NULL);
} }
} }
@@ -537,7 +538,7 @@ struct partial {
long long extent = 0; long long extent = 0;
long long clustered = 0; long long clustered = 0;
std::set<std::string> need_tilestats; std::set<std::string> need_tilestats;
std::map<std::string, accum_state> attribute_accum_state; std::unordered_map<std::string, accum_state> attribute_accum_state;
}; };
struct partial_arg { struct partial_arg {
@@ -1404,7 +1405,7 @@ struct write_tile_args {
size_t feature_count_out = 0; size_t feature_count_out = 0;
const char *prefilter = NULL; const char *prefilter = NULL;
const char *postfilter = NULL; const char *postfilter = NULL;
std::map<std::string, attribute_op> const *attribute_accum = NULL; std::unordered_map<std::string, attribute_op> const *attribute_accum = NULL;
bool still_dropping = false; bool still_dropping = false;
int wrote_zoom = 0; int wrote_zoom = 0;
size_t tiling_seg = 0; size_t tiling_seg = 0;
@@ -1593,7 +1594,7 @@ serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos
} }
if (filter != NULL) { if (filter != NULL) {
std::map<std::string, mvt_value> attributes; std::unordered_map<std::string, mvt_value> attributes;
std::string layername = (*layer_unmaps)[sf.segment][sf.layer]; std::string layername = (*layer_unmaps)[sf.segment][sf.layer];
std::set<std::string> exclude_attributes; std::set<std::string> exclude_attributes;
@@ -1627,11 +1628,11 @@ serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos
v.type = mvt_string; v.type = mvt_string;
if (sf.t == mvt_point) { if (sf.t == mvt_point) {
v.string_value = "Point"; v.set_string_value("Point");
} else if (sf.t == mvt_linestring) { } else if (sf.t == mvt_linestring) {
v.string_value = "LineString"; v.set_string_value("LineString");
} else if (sf.t == mvt_polygon) { } else if (sf.t == mvt_polygon) {
v.string_value = "Polygon"; v.set_string_value("Polygon");
} }
attributes.insert(std::pair<std::string, mvt_value>("$type", v)); attributes.insert(std::pair<std::string, mvt_value>("$type", v));
@@ -1846,7 +1847,7 @@ void promote_attribute(std::string const &key, partial &p, char *stringpool, lon
} }
} }
void preserve_attributes(std::map<std::string, attribute_op> const *attribute_accum, serial_feature &sf, char *stringpool, long long *pool_off, partial &p) { void preserve_attributes(std::unordered_map<std::string, attribute_op> const *attribute_accum, serial_feature &sf, char *stringpool, long long *pool_off, partial &p) {
for (size_t i = 0; i < sf.keys.size(); i++) { for (size_t i = 0; i < sf.keys.size(); i++) {
std::string key = stringpool + pool_off[sf.segment] + sf.keys[i] + 1; std::string key = stringpool + pool_off[sf.segment] + sf.keys[i] + 1;
@@ -3117,7 +3118,7 @@ void *run_thread(void *vargs) {
return NULL; return NULL;
} }
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos) { int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos) {
last_progress = 0; last_progress = 0;
// The existing layermaps are one table per input thread. // The existing layermaps are one table per input thread.
tile.hpp
+1 -1
View File
@@ -55,7 +55,7 @@ struct strategy {
// long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy); // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos); int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);
write_json.cpp
+19 -10
View File
@@ -336,31 +336,40 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y
const char *key = layer.keys[feat.tags[t]].c_str(); const char *key = layer.keys[feat.tags[t]].c_str();
mvt_value const &val = layer.values[feat.tags[t + 1]]; mvt_value const &val = layer.values[feat.tags[t + 1]];
if (val.type == mvt_string) { switch (val.type) {
case mvt_string:
state.json_write_string(key); state.json_write_string(key);
state.json_write_string(val.string_value); state.json_write_string(val.get_string_value());
} else if (val.type == mvt_int) { break;
case mvt_int:
state.json_write_string(key); state.json_write_string(key);
state.json_write_signed(val.numeric_value.int_value); state.json_write_signed(val.numeric_value.int_value);
} else if (val.type == mvt_double) { break;
case mvt_double:
state.json_write_string(key); state.json_write_string(key);
state.json_write_number(val.numeric_value.double_value); state.json_write_number(val.numeric_value.double_value);
} else if (val.type == mvt_float) { break;
case mvt_float:
state.json_write_string(key); state.json_write_string(key);
state.json_write_number(val.numeric_value.float_value); state.json_write_number(val.numeric_value.float_value);
} else if (val.type == mvt_sint) { break;
case mvt_sint:
state.json_write_string(key); state.json_write_string(key);
state.json_write_signed(val.numeric_value.sint_value); state.json_write_signed(val.numeric_value.sint_value);
} else if (val.type == mvt_uint) { break;
case mvt_uint:
state.json_write_string(key); state.json_write_string(key);
state.json_write_unsigned(val.numeric_value.uint_value); state.json_write_unsigned(val.numeric_value.uint_value);
} else if (val.type == mvt_bool) { break;
case mvt_bool:
state.json_write_string(key); state.json_write_string(key);
state.json_write_bool(val.numeric_value.bool_value); state.json_write_bool(val.numeric_value.bool_value);
} else if (val.type == mvt_null) { break;
case mvt_null:
state.json_write_string(key); state.json_write_string(key);
state.json_write_null(); state.json_write_null();
} else { break;
default:
fprintf(stderr, "Internal error: property with unknown type\n"); fprintf(stderr, "Internal error: property with unknown type\n");
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }