From 358f0193721a22135907297568d34ef7fc6a10d8 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 22 Apr 2016 15:06:26 -0700 Subject: [PATCH] Factor out vector tile writing --- decode.cc | 9 ++++ protobuf.cc | 121 ++++++++++++++++++++++++++++++++++++++++++++++- protobuf.hh | 1 + tile.cc | 132 +++++++++++++++++----------------------------------- 4 files changed, 172 insertions(+), 91 deletions(-) diff --git a/decode.cc b/decode.cc index 0bc8e5df..d1f76ab8 100644 --- a/decode.cc +++ b/decode.cc @@ -96,6 +96,15 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) { printf(", "); } + if (feat.tags[t] >= layer.keys.size()) { + fprintf(stderr, "Error: out of bounds feature key\n"); + exit(EXIT_FAILURE); + } + if (feat.tags[t + 1] >= layer.values.size()) { + fprintf(stderr, "Error: out of bounds feature value\n"); + exit(EXIT_FAILURE); + } + const char *key = layer.keys[feat.tags[t]].c_str(); pb_value const &val = layer.values[feat.tags[t + 1]]; diff --git a/protobuf.cc b/protobuf.cc index 8b2dcf31..5a894522 100644 --- a/protobuf.cc +++ b/protobuf.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -94,7 +95,7 @@ bool pb_decode(std::string &message, pb_tile &out) { return false; } - for (int l = 0; l < tile.layers_size(); l++) { + for (size_t l = 0; l < tile.layers_size(); l++) { mapnik::vector::tile_layer layer = tile.layers(l); pb_layer pbl; pbl.extent = layer.extent(); @@ -134,7 +135,7 @@ bool pb_decode(std::string &message, pb_tile &out) { pbl.values.push_back(pbv); } - for (int f = 0; f < layer.features_size(); f++) { + for (size_t f = 0; f < layer.features_size(); f++) { mapnik::vector::tile_feature feat = layer.features(f); pb_feature pbf; pbf.type = feat.type(); @@ -170,3 +171,119 @@ bool pb_decode(std::string &message, pb_tile &out) { return true; } + +std::string pb_encode(pb_tile &in) { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + mapnik::vector::tile tile; + + for (size_t i = 0; i < in.layers.size(); i++) { + mapnik::vector::tile_layer *layer = tile.add_layers(); + + layer->set_name(in.layers[i].name); + layer->set_version(in.layers[i].version); + layer->set_extent(in.layers[i].extent); + + for (size_t k = 0; k < in.layers[i].keys.size(); k++) { + layer->add_keys(in.layers[i].keys[k]); + } + for (size_t v = 0; v < in.layers[i].values.size(); v++) { + mapnik::vector::tile_value *tv = layer->add_values(); + pb_value &pbv = in.layers[i].values[v]; + + if (pbv.type == pb_string) { + tv->set_string_value(pbv.string_value); + } else if (pbv.type == pb_float) { + tv->set_float_value(pbv.numeric_value.float_value); + } else if (pbv.type == pb_double) { + tv->set_double_value(pbv.numeric_value.double_value); + } else if (pbv.type == pb_int) { + tv->set_int_value(pbv.numeric_value.int_value); + } else if (pbv.type == pb_uint) { + tv->set_uint_value(pbv.numeric_value.uint_value); + } else if (pbv.type == pb_sint) { + tv->set_sint_value(pbv.numeric_value.sint_value); + } else if (pbv.type == pb_bool) { + tv->set_bool_value(pbv.numeric_value.bool_value); + } + } + + for (size_t f = 0; f < in.layers[i].features.size(); f++) { + mapnik::vector::tile_feature *feature = layer->add_features(); + if (feature == NULL) { + perror("add feature"); + exit(EXIT_FAILURE); + } + + int type = in.layers[i].features[f].type; + if (type == pb_point) { + feature->set_type(mapnik::vector::tile::Point); + } else if (type == pb_linestring) { + feature->set_type(mapnik::vector::tile::LineString); + } else if (type == pb_polygon) { + feature->set_type(mapnik::vector::tile::Polygon); + } else { + fprintf(stderr, "Corrupt geometry type\n"); + exit(EXIT_FAILURE); + } + + for (size_t t = 0; t < in.layers[i].features[f].tags.size(); t++) { + feature->add_tags(in.layers[i].features[f].tags[t]); + } + + int px = 0, py = 0; + int cmd_idx = -1; + int cmd = -1; + int length = 0; + + std::vector &geom = in.layers[i].features[f].geometry; + + for (size_t g = 0; g < geom.size(); g++) { + int op = geom[g].op; + + if (op != cmd) { + if (cmd_idx >= 0) { + feature->set_geometry(cmd_idx, (length << 3) | (cmd & ((1 << 3) - 1))); + } + + cmd = op; + length = 0; + cmd_idx = feature->geometry_size(); + feature->add_geometry(0); + } + + if (op == pb_moveto || op == pb_lineto) { + long long wwx = geom[g].x; + long long wwy = geom[g].y; + + int dx = wwx - px; + int dy = wwy - py; + + if (feature != NULL) { + feature->add_geometry((dx << 1) ^ (dx >> 31)); + feature->add_geometry((dy << 1) ^ (dy >> 31)); + } + + px = wwx; + py = wwy; + length++; + } else if (op == pb_closepath) { + length++; + } else { + fprintf(stderr, "\nInternal error: corrupted geometry\n"); + exit(EXIT_FAILURE); + } + } + + if (cmd_idx >= 0) { + feature->set_geometry(cmd_idx, (length << 3) | (cmd & ((1 << 3) - 1))); + } + } + } + + std::string s, compressed; + tile.SerializeToString(&s); + compress(s, compressed); + + return compressed; +} diff --git a/protobuf.hh b/protobuf.hh index 2526dd68..b378af7b 100644 --- a/protobuf.hh +++ b/protobuf.hh @@ -60,3 +60,4 @@ struct pb_tile { }; bool pb_decode(std::string &message, pb_tile &out); +std::string pb_encode(pb_tile &in); diff --git a/tile.cc b/tile.cc index 352bc389..b581c5a2 100644 --- a/tile.cc +++ b/tile.cc @@ -38,68 +38,24 @@ extern "C" { pthread_mutex_t db_lock = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t var_lock = PTHREAD_MUTEX_INITIALIZER; -int to_feature(drawvec &geom, mapnik::vector::tile_feature *feature) { - int px = 0, py = 0; - int cmd_idx = -1; - int cmd = -1; - int length = 0; - int drew = 0; - int i; +std::vector to_feature(drawvec &geom) { + std::vector out; - int n = geom.size(); - for (i = 0; i < n; i++) { - int op = geom[i].op; + for (size_t i = 0; i < geom.size(); i++) { + out.push_back(pb_geometry(geom[i].op, geom[i].x, geom[i].y)); + } - if (op != cmd) { - if (cmd_idx >= 0) { - if (feature != NULL) { - feature->set_geometry(cmd_idx, (length << CMD_BITS) | (cmd & ((1 << CMD_BITS) - 1))); - } - } + return out; +} - cmd = op; - length = 0; - - if (feature != NULL) { - cmd_idx = feature->geometry_size(); - feature->add_geometry(0); - } - } - - if (op == VT_MOVETO || op == VT_LINETO) { - long long wwx = geom[i].x; - long long wwy = geom[i].y; - - int dx = wwx - px; - int dy = wwy - py; - - if (feature != NULL) { - feature->add_geometry((dx << 1) ^ (dx >> 31)); - feature->add_geometry((dy << 1) ^ (dy >> 31)); - } - - px = wwx; - py = wwy; - length++; - - if (op == VT_LINETO && (dx != 0 || dy != 0)) { - drew = 1; - } - } else if (op == VT_CLOSEPATH) { - length++; - } else { - fprintf(stderr, "\nInternal error: corrupted geometry\n"); - exit(EXIT_FAILURE); +bool draws_something(drawvec &geom) { + for (size_t i = 1; i < geom.size(); i++) { + if (geom[i].op == VT_LINETO && (geom[i].x != geom[i - 1].x || geom[i].y != geom[i - 1].y)) { + return true; } } - if (cmd_idx >= 0) { - if (feature != NULL) { - feature->set_geometry(cmd_idx, (length << CMD_BITS) | (cmd & ((1 << CMD_BITS) - 1))); - } - } - - return drew; + return false; } int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2); @@ -247,8 +203,8 @@ static int is_integer(const char *s, long long *v) { return 1; } -mapnik::vector::tile create_tile(char **layernames, int line_detail, std::vector > &features, long long *count, struct pool **keys, struct pool **values, int nlayers) { - mapnik::vector::tile tile; +pb_tile create_tile(char **layernames, int line_detail, std::vector > &features, long long *count, struct pool **keys, struct pool **values, int nlayers) { + pb_tile tile; int i; for (i = 0; i < nlayers; i++) { @@ -256,61 +212,63 @@ mapnik::vector::tile create_tile(char **layernames, int line_detail, std::vector continue; } - mapnik::vector::tile_layer *layer = tile.add_layers(); + pb_layer layer; - layer->set_name(layernames[i]); - layer->set_version(1); - layer->set_extent(1 << line_detail); + layer.name = layernames[i]; + layer.version = 1; + layer.extent = 1 << line_detail; for (size_t x = 0; x < features[i].size(); x++) { if (features[i][x].type == VT_LINE || features[i][x].type == VT_POLYGON) { features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type, 0); } - mapnik::vector::tile_feature *feature = layer->add_features(); + pb_feature pbf; + pbf.type = features[i][x].type; - if (features[i][x].type == VT_POINT) { - feature->set_type(mapnik::vector::tile::Point); - } else if (features[i][x].type == VT_LINE) { - feature->set_type(mapnik::vector::tile::LineString); - } else if (features[i][x].type == VT_POLYGON) { - feature->set_type(mapnik::vector::tile::Polygon); - } else { - feature->set_type(mapnik::vector::tile::Unknown); - } - - to_feature(features[i][x].geom, feature); + pbf.geometry = to_feature(features[i][x].geom); *count += features[i][x].geom.size(); for (size_t y = 0; y < features[i][x].meta.size(); y++) { - feature->add_tags(features[i][x].meta[y]); + pbf.tags.push_back(features[i][x].meta[y]); } + + layer.features.push_back(pbf); } struct pool_val *pv; for (pv = keys[i]->head; pv != NULL; pv = pv->next) { - layer->add_keys(pv->s, strlen(pv->s)); + layer.keys.push_back(std::string(pv->s, strlen(pv->s))); } for (pv = values[i]->head; pv != NULL; pv = pv->next) { - mapnik::vector::tile_value *tv = layer->add_values(); + pb_value tv; if (pv->type == VT_NUMBER) { long long v; if (is_integer(pv->s, &v)) { if (v >= 0) { - tv->set_int_value(v); + tv.type = pb_int; + tv.numeric_value.int_value = v; } else { - tv->set_sint_value(v); + tv.type = pb_sint; + tv.numeric_value.sint_value = v; } } else { - tv->set_double_value(atof(pv->s)); + tv.type = pb_double; + tv.numeric_value.double_value = atof(pv->s); } } else if (pv->type == VT_BOOLEAN) { - tv->set_bool_value(pv->s[0] == 't'); + tv.type = pb_bool; + tv.numeric_value.bool_value = (pv->s[0] == 't'); } else { - tv->set_string_value(pv->s); + tv.type = pb_string; + tv.string_value = pv->s; } + + layer.values.push_back(tv); } + + tile.layers.push_back(layer); } return tile; @@ -890,7 +848,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s // A complex polygon may have been split up into multiple geometries. // Break them out into multiple features if necessary. for (size_t j = 0; j < geoms.size(); j++) { - if (t == VT_POINT || to_feature(geoms[j], NULL)) { + if (t == VT_POINT || draws_something(geoms[j])) { struct coalesce c; char *meta = partials[i].meta; @@ -986,7 +944,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s return -1; } - mapnik::vector::tile tile = create_tile(layernames, line_detail, features, &count, keys, values, nlayers); + pb_tile tile = create_tile(layernames, line_detail, features, &count, keys, values, nlayers); int i; for (i = 0; i < nlayers; i++) { @@ -994,11 +952,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s pool_free(&values1[i]); } - std::string s; - std::string compressed; - - tile.SerializeToString(&s); - compress(s, compressed); + std::string compressed = pb_encode(tile); if (compressed.size() > 500000 && !prevent[P_KILOBYTE_LIMIT]) { if (!quiet) {