From e10a71e15291c81fa4f5abe0262878a7055156fb Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Mon, 25 Apr 2016 16:19:52 -0700 Subject: [PATCH] Use the mvt constant pool when tiling --- tile.cc | 196 ++++++++++++++++++++++++-------------------------------- 1 file changed, 82 insertions(+), 114 deletions(-) diff --git a/tile.cc b/tile.cc index 0c2f77ad..4bbf23a3 100644 --- a/tile.cc +++ b/tile.cc @@ -59,11 +59,14 @@ bool draws_something(drawvec &geom) { } int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2); +static int is_integer(const char *s, long long *v); struct coalesce { int type; drawvec geom; - std::vector meta; + int segment; + int m; + char *meta; unsigned long long index; unsigned long long index2; bool coalesced; @@ -94,6 +97,8 @@ int coalcmp(const void *v1, const void *v2) { return cmp; } +#if 0 /* XXX */ + for (size_t i = 0; i < c1->meta.size() && i < c2->meta.size(); i++) { cmp = c1->meta[i] - c2->meta[i]; @@ -109,6 +114,9 @@ int coalcmp(const void *v1, const void *v2) { } else { return 0; } +#endif + + return 0; } int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2) { @@ -131,38 +139,65 @@ int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2) { return cmp; } -struct pool_val *retrieve_string(char **f, struct pool *p, char *stringpool) { - struct pool_val *ret; +mvt_value retrieve_string(char **f, char *stringpool, int *otype) { long long off; - deserialize_long_long(f, &off); - ret = pool(p, stringpool + off + 1, stringpool[off]); - return ret; + int type = stringpool[off]; + char *s = stringpool + off + 1; + + if (otype != NULL) { + *otype = type; + } + + mvt_value tv; + if (type == VT_NUMBER) { + long long v; + if (is_integer(s, &v)) { + if (v >= 0) { + tv.type = mvt_int; + tv.numeric_value.int_value = v; + } else { + tv.type = mvt_sint; + tv.numeric_value.sint_value = v; + } + } else { + tv.type = mvt_double; + tv.numeric_value.double_value = atof(s); + } + } else if (type == VT_BOOLEAN) { + tv.type = mvt_bool; + tv.numeric_value.bool_value = (s[0] == 't'); + } else { + tv.type = mvt_string; + tv.string_value = s; + } + + return tv; } -void decode_meta(int m, char **meta, char *stringpool, struct pool *keys, struct pool *values, struct pool *file_keys, std::vector *intmeta) { +void decode_meta(int m, char **meta, char *stringpool, mvt_layer &layer, mvt_feature &feature, struct pool *file_keys) { int i; for (i = 0; i < m; i++) { - struct pool_val *key = retrieve_string(meta, keys, stringpool); - struct pool_val *value = retrieve_string(meta, values, stringpool); + int otype; + mvt_value key = retrieve_string(meta, stringpool, NULL); + mvt_value value = retrieve_string(meta, stringpool, &otype); - intmeta->push_back(key->n); - intmeta->push_back(value->n); + layer.tag(feature, key.string_value, value); - if (!is_pooled(file_keys, key->s, value->type)) { + if (!is_pooled(file_keys, key.string_value.c_str(), otype)) { if (pthread_mutex_lock(&var_lock) != 0) { perror("pthread_mutex_lock"); exit(EXIT_FAILURE); } // Dup to retain after munmap - char *copy = strdup(key->s); + char *copy = strdup(key.string_value.c_str()); if (copy == NULL) { perror("Out of memory"); exit(EXIT_FAILURE); } - pool(file_keys, copy, value->type); + pool(file_keys, copy, otype); if (pthread_mutex_unlock(&var_lock) != 0) { perror("pthread_mutex_unlock"); @@ -203,77 +238,6 @@ static int is_integer(const char *s, long long *v) { return 1; } -mvt_tile create_tile(char **layernames, int line_detail, std::vector > &features, long long *count, struct pool **keys, struct pool **values, int nlayers) { - mvt_tile tile; - - int i; - for (i = 0; i < nlayers; i++) { - if (features[i].size() == 0) { - continue; - } - - mvt_layer layer; - - layer.name = layernames[i]; - layer.version = 2; - layer.extent = 1 << line_detail; - - for (size_t x = 0; x < features[i].size(); x++) { - if (features[i][x].type == VT_LINE || features[i][x].type == VT_POLYGON) { - features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type, 0); - } - - mvt_feature pbf; - pbf.type = features[i][x].type; - - pbf.geometry = to_feature(features[i][x].geom); - *count += features[i][x].geom.size(); - - for (size_t y = 0; y < features[i][x].meta.size(); y++) { - pbf.tags.push_back(features[i][x].meta[y]); - } - - layer.features.push_back(pbf); - } - - struct pool_val *pv; - for (pv = keys[i]->head; pv != NULL; pv = pv->next) { - layer.keys.push_back(std::string(pv->s, strlen(pv->s))); - } - for (pv = values[i]->head; pv != NULL; pv = pv->next) { - mvt_value tv; - - if (pv->type == VT_NUMBER) { - long long v; - if (is_integer(pv->s, &v)) { - if (v >= 0) { - tv.type = mvt_int; - tv.numeric_value.int_value = v; - } else { - tv.type = mvt_sint; - tv.numeric_value.sint_value = v; - } - } else { - tv.type = mvt_double; - tv.numeric_value.double_value = atof(pv->s); - } - } else if (pv->type == VT_BOOLEAN) { - tv.type = mvt_bool; - tv.numeric_value.bool_value = (pv->s[0] == 't'); - } else { - tv.type = mvt_string; - tv.string_value = pv->s; - } - - layer.values.push_back(tv); - } - - tile.layers.push_back(layer); - } - - return tile; -} - struct sll { char *name; long long val; @@ -557,17 +521,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s // This only loops if the tile data didn't fit, in which case the detail // goes down and the progress indicator goes backward for the next try. for (line_detail = detail; line_detail >= min_detail || line_detail == detail; line_detail--, oprogress = 0) { - struct pool keys1[nlayers], values1[nlayers]; - struct pool *keys[nlayers], *values[nlayers]; - int i; - for (i = 0; i < nlayers; i++) { - pool_init(&keys1[i], 0); - pool_init(&values1[i], 0); - - keys[i] = &keys1[i]; - values[i] = &values1[i]; - } - long long count = 0; double accum_area = 0; @@ -588,7 +541,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s std::vector partials; std::vector > features; - for (i = 0; i < nlayers; i++) { + for (size_t i = 0; i < nlayers; i++) { features.push_back(std::vector()); } @@ -840,7 +793,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s partials[i].geoms.clear(); // avoid keeping two copies in memory long long layer = partials[i].layer; signed char t = partials[i].t; - int segment = partials[i].segment; long long original_seq = partials[i].original_seq; // A complex polygon may have been split up into multiple geometries. @@ -848,7 +800,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s for (size_t j = 0; j < geoms.size(); j++) { if (t == VT_POINT || draws_something(geoms[j])) { struct coalesce c; - char *meta = partials[i].meta; c.type = t; c.index = partials[i].index; @@ -856,8 +807,10 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s c.geom = geoms[j]; c.coalesced = false; c.original_seq = original_seq; + c.m = partials[i].m; + c.segment = partials[i].segment; + c.meta = partials[i].meta; - decode_meta(partials[i].m, &meta, stringpool + pool_off[segment], keys[layer], values[layer], file_keys[layer], &c.meta); features[layer].push_back(c); } } @@ -925,6 +878,35 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s } } + mvt_tile tile; + + for (size_t j = 0; j < features.size(); j++) { + mvt_layer mvtl; + + mvtl.name = layernames[j]; + mvtl.version = 2; + mvtl.extent = 1 << line_detail; + + for (size_t x = 0; x < features[j].size(); x++) { + mvt_feature mvtf; + + if (features[j][x].type == VT_LINE || features[j][x].type == VT_POLYGON) { + features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type, 0); + } + + mvtf.type = features[j][x].type; + mvtf.geometry = to_feature(features[j][x].geom); + count += features[j][x].geom.size(); + + decode_meta(features[j][x].m, &features[j][x].meta, stringpool + pool_off[features[j][x].segment], mvtl, mvtf, file_keys[j]); + mvtl.features.push_back(mvtf); + } + + if (mvtl.features.size() > 0) { + tile.layers.push_back(mvtl); + } + } + if (z == 0 && unclipped_features < original_features / 2) { fprintf(stderr, "\n\nMore than half the features were clipped away at zoom level 0.\n"); fprintf(stderr, "Is your data in the wrong projection? It should be in WGS84/EPSG:4326.\n"); @@ -942,14 +924,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s return -1; } - mvt_tile tile = create_tile(layernames, line_detail, features, &count, keys, values, nlayers); - - int i; - for (i = 0; i < nlayers; i++) { - pool_free(&keys1[i]); - pool_free(&values1[i]); - } - std::string compressed = tile.encode(); if (compressed.size() > 500000 && !prevent[P_KILOBYTE_LIMIT]) { @@ -983,12 +957,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s return count; } } else { - int i; - for (i = 0; i < nlayers; i++) { - pool_free(&keys1[i]); - pool_free(&values1[i]); - } - return count; } }