From caafe2fe5cf260233cc91ed2ba9ab40e76f0b9e5 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 13:02:34 -0700 Subject: [PATCH 01/12] Checkpoint in copying data from one tileset to another --- Makefile | 5 +- tile-join.cc | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 tile-join.cc diff --git a/Makefile b/Makefile index c0653b1f..f181e634 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PREFIX ?= /usr/local MANDIR ?= /usr/share/man/man1/ -all: tippecanoe enumerate decode +all: tippecanoe enumerate decode tile-join docs: man/tippecanoe.1 @@ -33,6 +33,9 @@ enumerate: enumerate.o decode: decode.o vector_tile.pb.o projection.o g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 +tile-join: tile-join.o vector_tile.pb.o projection.o pool.o + g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 + libjsonpull.a: jsonpull.o ar rc $@ $^ ranlib $@ diff --git a/tile-join.cc b/tile-join.cc new file mode 100644 index 00000000..57fa9b41 --- /dev/null +++ b/tile-join.cc @@ -0,0 +1,172 @@ +#include +#include +#include +#include +#include +#include +#include +#include "vector_tile.pb.h" + +extern "C" { +#include "projection.h" +#include "pool.h" +} + +// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp +inline bool is_compressed(std::string const &data) { + return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); +} + +// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp +inline int decompress(std::string const &input, std::string &output) { + z_stream inflate_s; + inflate_s.zalloc = Z_NULL; + inflate_s.zfree = Z_NULL; + inflate_s.opaque = Z_NULL; + inflate_s.avail_in = 0; + inflate_s.next_in = Z_NULL; + if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) { + fprintf(stderr, "error: %s\n", inflate_s.msg); + } + inflate_s.next_in = (Bytef *) input.data(); + inflate_s.avail_in = input.size(); + size_t length = 0; + do { + output.resize(length + 2 * input.size()); + inflate_s.avail_out = 2 * input.size(); + inflate_s.next_out = (Bytef *) (output.data() + length); + int ret = inflate(&inflate_s, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + fprintf(stderr, "error: %s\n", inflate_s.msg); + return 0; + } + + length += (2 * input.size() - inflate_s.avail_out); + } while (inflate_s.avail_out == 0); + inflateEnd(&inflate_s); + output.resize(length); + return 1; +} + +int dezig(unsigned n) { + return (n >> 1) ^ (-(n & 1)); +} + +void handle(std::string message, int z, unsigned x, unsigned y, struct pool *file_keys) { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp + mapnik::vector::tile tile; + mapnik::vector::tile outtile; + + if (is_compressed(message)) { + std::string uncompressed; + decompress(message, uncompressed); + if (!tile.ParseFromString(uncompressed)) { + fprintf(stderr, "Couldn't decompress tile %d/%u/%u\n", z, x, y); + exit(EXIT_FAILURE); + } + } else if (!tile.ParseFromString(message)) { + fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y); + exit(EXIT_FAILURE); + } + + for (int l = 0; l < tile.layers_size(); l++) { + mapnik::vector::tile_layer layer = tile.layers(l); + mapnik::vector::tile_layer *outlayer = outtile.add_layers(); + + outlayer->set_name(layer.name()); + outlayer->set_version(layer.version()); + outlayer->set_extent(layer.extent()); + + for (int f = 0; f < layer.features_size(); f++) { + mapnik::vector::tile_feature feat = layer.features(f); + mapnik::vector::tile_feature *outfeature = outlayer->add_features(); + + outfeature->set_type(feat.type()); + + for (int g = 0; g < feat.geometry_size(); g++) { + outfeature->add_geometry(feat.geometry(g)); + } + } + } +} + +void decode(char *fname, char *map, struct pool *file_keys) { + sqlite3 *db; + + if (sqlite3_open(fname, &db) != SQLITE_OK) { + fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + + const char *sql = "SELECT zoom_level, tile_column, tile_row, tile_data from tiles;"; + sqlite3_stmt *stmt; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { + fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } + + while (sqlite3_step(stmt) == SQLITE_ROW) { + long long zoom = sqlite3_column_int(stmt, 0); + long long x = sqlite3_column_int(stmt, 1); + long long y = sqlite3_column_int(stmt, 2); + y = (1LL << zoom) - 1 - y; + + int len = sqlite3_column_bytes(stmt, 3); + const char *s = (const char *) sqlite3_column_blob(stmt, 3); + + printf("found %lld/%lld/%lld\n", zoom, x, y); + + handle(std::string(s, len), zoom, x, y, file_keys); + } + + sqlite3_finalize(stmt); + + if (sqlite3_close(db) != SQLITE_OK) { + fprintf(stderr, "%s: could not close database: %s\n", fname, sqlite3_errmsg(db)); + exit(EXIT_FAILURE); + } +} + +void usage(char **argv) { + fprintf(stderr, "Usage: %s [-f] -o new.mbtiles source.mbtiles map.csv\n", argv[0]); + exit(EXIT_FAILURE); +} + +int main(int argc, char **argv) { + char *outfile = NULL; + int force = 0; + + extern int optind; + extern char *optarg; + int i; + + while ((i = getopt(argc, argv, "fo:")) != -1) { + switch (i) { + case 'o': + outfile = optarg; + break; + + case 'f': + force = 1; + break; + + default: + usage(argv); + } + } + + if (argc != optind + 2 || outfile == NULL) { + usage(argv); + } + + struct pool file_keys; + pool_init(&file_keys, 0); + + decode(argv[optind], argv[optind + 1], &file_keys); + + pool_free(&file_keys); + + return 0; +} From 224321f6c855f9457642bb18c05aac42f56900f1 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 13:42:24 -0700 Subject: [PATCH 02/12] Look up layers by name because the source mbtiles might be inconsistent --- tile-join.cc | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index 57fa9b41..15b9f005 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -52,7 +52,7 @@ int dezig(unsigned n) { return (n >> 1) ^ (-(n & 1)); } -void handle(std::string message, int z, unsigned x, unsigned y, struct pool *file_keys) { +void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers) { GOOGLE_PROTOBUF_VERIFY_VERSION; // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp @@ -79,6 +79,23 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool *fil outlayer->set_version(layer.version()); outlayer->set_extent(layer.extent()); + const char *ln = layer.name().c_str(); + + int ll; + for (ll = 0; ll < *nlayers; ll++) { + if (strcmp((*layernames)[ll], ln) == 0) { + break; + } + } + if (ll == *nlayers) { + *file_keys = (struct pool *) realloc(*file_keys, (ll + 1) * sizeof(struct pool)); + *layernames = (char **) realloc(*layernames, (ll + 1) * sizeof(char *)); + + pool_init(file_keys[ll], 0); + (*layernames)[ll] = strdup(ln); + *nlayers = ll + 1; + } + for (int f = 0; f < layer.features_size(); f++) { mapnik::vector::tile_feature feat = layer.features(f); mapnik::vector::tile_feature *outfeature = outlayer->add_features(); @@ -92,7 +109,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool *fil } } -void decode(char *fname, char *map, struct pool *file_keys) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -118,7 +135,7 @@ void decode(char *fname, char *map, struct pool *file_keys) { printf("found %lld/%lld/%lld\n", zoom, x, y); - handle(std::string(s, len), zoom, x, y, file_keys); + handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers); } sqlite3_finalize(stmt); @@ -161,12 +178,15 @@ int main(int argc, char **argv) { usage(argv); } - struct pool file_keys; - pool_init(&file_keys, 0); + struct pool *file_keys = NULL; + char **layernames = NULL; + int nlayers = 0; - decode(argv[optind], argv[optind + 1], &file_keys); + decode(argv[optind], argv[optind + 1], &file_keys, &layernames, &nlayers); - pool_free(&file_keys); + for (i = 0; i < nlayers; i++) { + printf("%s\n", layernames[i]); + } return 0; } From 863c9a59291789244ef697b9ac37b55c52326192 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 14:27:39 -0700 Subject: [PATCH 03/12] Retrieving keys and values --- pool.c | 4 ++-- pool.h | 2 +- tile-join.cc | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/pool.c b/pool.c index 74dd536c..cc55df9f 100644 --- a/pool.c +++ b/pool.c @@ -5,7 +5,7 @@ #define POOL_WIDTH 256 -static int hash(char *s) { +static int hash(const char *s) { int h = 0; for (; *s; s++) { h = h * 37 + *s; @@ -57,7 +57,7 @@ struct pool_val *pool(struct pool *p, char *s, int type) { return *v; } -int is_pooled(struct pool *p, char *s, int type) { +int is_pooled(struct pool *p, const char *s, int type) { int h = hash(s); struct pool_val **v = &(p->vals[h]); diff --git a/pool.h b/pool.h index 147e8c81..683b5511 100644 --- a/pool.h +++ b/pool.h @@ -21,4 +21,4 @@ struct pool_val *pool(struct pool *p, char *s, int type); void pool_free(struct pool *p); void pool_free_strings(struct pool *p); void pool_init(struct pool *p, int n); -int is_pooled(struct pool *p, char *s, int type); +int is_pooled(struct pool *p, const char *s, int type); diff --git a/tile-join.cc b/tile-join.cc index 15b9f005..af02b0d5 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -1,3 +1,6 @@ +// for vasprintf() on Linux +#define _GNU_SOURCE + #include #include #include @@ -6,6 +9,7 @@ #include #include #include "vector_tile.pb.h" +#include "tile.h" extern "C" { #include "projection.h" @@ -105,6 +109,50 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi for (int g = 0; g < feat.geometry_size(); g++) { outfeature->add_geometry(feat.geometry(g)); } + + for (int t = 0; t + 1 < feat.tags_size(); t += 2) { + if (feat.tags(t) >= layer.keys_size() || feat.tags(t + 1) >= layer.values_size()) { + printf("out of range: %d=%d\n", feat.tags(t), feat.tags(t + 1)); + continue; + } + + const char *key = layer.keys(feat.tags(t)).c_str(); + mapnik::vector::tile_value const &val = layer.values(feat.tags(t + 1)); + char *value; + int type; + + if (val.has_string_value()) { + value = strdup(val.string_value().c_str()); + type = VT_STRING; + } else if (val.has_int_value()) { + asprintf(&value, "%lld", val.int_value()); + type = VT_NUMBER; + } else if (val.has_double_value()) { + asprintf(&value, "%g", val.double_value()); + type = VT_NUMBER; + } else if (val.has_float_value()) { + asprintf(&value, "%g", val.float_value()); + type = VT_NUMBER; + } else if (val.has_bool_value()) { + asprintf(&value, "%s", val.bool_value() ? "true" : "false"); + type = VT_BOOLEAN; + } else if (val.has_sint_value()) { + asprintf(&value, "%lld", val.sint_value()); + type = VT_NUMBER; + } else if (val.has_uint_value()) { + asprintf(&value, "%llu", val.uint_value()); + type = VT_NUMBER; + } else { + continue; + } + + if (!is_pooled(file_keys[ll], key, type)) { + pool(file_keys[ll], strdup(key), type); + } + + printf("%d: %s=%s\n", type, key, value); + free(value); + } } } } From ed378681e455b3190c728162ba58ecd74ee1868d Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 14:50:26 -0700 Subject: [PATCH 04/12] Making the string pool in the destination tile --- pool.c | 4 ++-- pool.h | 4 ++-- tile-join.cc | 26 +++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/pool.c b/pool.c index cc55df9f..4641afc5 100644 --- a/pool.c +++ b/pool.c @@ -14,7 +14,7 @@ static int hash(const char *s) { return h; } -struct pool_val *pool(struct pool *p, char *s, int type) { +struct pool_val *pool(struct pool *p, const char *s, int type) { int h = hash(s); struct pool_val **v = &(p->vals[h]); @@ -83,7 +83,7 @@ int is_pooled(struct pool *p, const char *s, int type) { void pool_free1(struct pool *p, void (*func)(void *)) { while (p->head != NULL) { if (func != NULL) { - func(p->head->s); + func((void *) p->head->s); } struct pool_val *next = p->head->next; diff --git a/pool.h b/pool.h index 683b5511..100dd76f 100644 --- a/pool.h +++ b/pool.h @@ -1,5 +1,5 @@ struct pool_val { - char *s; + const char *s; int type; int n; @@ -17,7 +17,7 @@ struct pool { int n; }; -struct pool_val *pool(struct pool *p, char *s, int type); +struct pool_val *pool(struct pool *p, const char *s, int type); void pool_free(struct pool *p); void pool_free_strings(struct pool *p); void pool_init(struct pool *p, int n); diff --git a/tile-join.cc b/tile-join.cc index af02b0d5..89f9e8c3 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -100,6 +100,10 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi *nlayers = ll + 1; } + struct pool keys, values; + pool_init(&keys, 0); + pool_init(&values, 0); + for (int f = 0; f < layer.features_size(); f++) { mapnik::vector::tile_feature feat = layer.features(f); mapnik::vector::tile_feature *outfeature = outlayer->add_features(); @@ -150,10 +154,30 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi pool(file_keys[ll], strdup(key), type); } - printf("%d: %s=%s\n", type, key, value); + struct pool_val *k, *v; + + if (is_pooled(&keys, key, VT_STRING)) { + k = pool(&keys, key, VT_STRING); + } else { + k = pool(&keys, strdup(key), VT_STRING); + } + + if (is_pooled(&values, value, type)) { + v = pool(&values, value, type); + } else { + v = pool(&values, strdup(value), type); + } + + outfeature->add_tags(k->n); + outfeature->add_tags(v->n); + + printf("%d: %s=%s %d=%d\n", type, key, value, k->n, v->n); free(value); } } + + pool_free_strings(&keys); + pool_free_strings(&values); } } From 2e3f03172e2749a1fba5909cb60300192c2a9fea Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 15:01:34 -0700 Subject: [PATCH 05/12] Write out the JSON part of the metadata --- Makefile | 2 +- tile-join.cc | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f181e634..0538346a 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ enumerate: enumerate.o decode: decode.o vector_tile.pb.o projection.o g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 -tile-join: tile-join.o vector_tile.pb.o projection.o pool.o +tile-join: tile-join.o vector_tile.pb.o projection.o pool.o mbtiles.o g++ $(PG) $(LIBS) -O3 -g -Wall -o $@ $^ -lm -lz -lprotobuf-lite -lsqlite3 libjsonpull.a: jsonpull.o diff --git a/tile-join.cc b/tile-join.cc index 89f9e8c3..71ff9fc9 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -14,6 +14,7 @@ extern "C" { #include "projection.h" #include "pool.h" +#include "mbtiles.h" } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp @@ -170,8 +171,6 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi outfeature->add_tags(k->n); outfeature->add_tags(v->n); - - printf("%d: %s=%s %d=%d\n", type, key, value, k->n, v->n); free(value); } } @@ -205,7 +204,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int len = sqlite3_column_bytes(stmt, 3); const char *s = (const char *) sqlite3_column_blob(stmt, 3); - printf("found %lld/%lld/%lld\n", zoom, x, y); + printf("%lld/%lld/%lld \r", zoom, x, y); handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers); } @@ -250,6 +249,12 @@ int main(int argc, char **argv) { usage(argv); } + if (force) { + unlink(outfile); + } + + sqlite3 *outdb = mbtiles_open(outfile, argv); + struct pool *file_keys = NULL; char **layernames = NULL; int nlayers = 0; @@ -260,5 +265,8 @@ int main(int argc, char **argv) { printf("%s\n", layernames[i]); } + mbtiles_write_metadata(outdb, outfile, layernames, 0, 0, 0, 0, 0, 0, 0, 0, &file_keys, nlayers); + mbtiles_close(outdb, argv); + return 0; } From 29fcbf244d23ed530b6c40fbf4a9cf66f799e83e Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 15:15:47 -0700 Subject: [PATCH 06/12] Write each tile to the destination tileset --- tile-join.cc | 62 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index 71ff9fc9..f73f4bb6 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -53,11 +53,35 @@ inline int decompress(std::string const &input, std::string &output) { return 1; } -int dezig(unsigned n) { - return (n >> 1) ^ (-(n & 1)); +// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp +static inline int compress(std::string const &input, std::string &output) { + z_stream deflate_s; + deflate_s.zalloc = Z_NULL; + deflate_s.zfree = Z_NULL; + deflate_s.opaque = Z_NULL; + deflate_s.avail_in = 0; + deflate_s.next_in = Z_NULL; + deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); + deflate_s.next_in = (Bytef *) input.data(); + deflate_s.avail_in = input.size(); + size_t length = 0; + do { + size_t increase = input.size() / 2 + 1024; + output.resize(length + increase); + deflate_s.avail_out = increase; + deflate_s.next_out = (Bytef *) (output.data() + length); + int ret = deflate(&deflate_s, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) { + return -1; + } + length += (increase - deflate_s.avail_out); + } while (deflate_s.avail_out == 0); + deflateEnd(&deflate_s); + output.resize(length); + return 0; } -void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers) { +void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { GOOGLE_PROTOBUF_VERIFY_VERSION; // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp @@ -175,12 +199,36 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi } } + struct pool_val *pv; + for (pv = keys.head; pv != NULL; pv = pv->next) { + outlayer->add_keys(pv->s, strlen(pv->s)); + } + for (pv = values.head; pv != NULL; pv = pv->next) { + mapnik::vector::tile_value *tv = outlayer->add_values(); + + if (pv->type == VT_NUMBER) { + tv->set_double_value(atof(pv->s)); + } else if (pv->type == VT_BOOLEAN) { + tv->set_bool_value(pv->s[0] == 't'); + } else { + tv->set_string_value(pv->s); + } + } + pool_free_strings(&keys); pool_free_strings(&values); } + + std::string s; + std::string compressed; + + outtile.SerializeToString(&s); + compress(s, compressed); + + mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } -void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -204,9 +252,9 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int len = sqlite3_column_bytes(stmt, 3); const char *s = (const char *) sqlite3_column_blob(stmt, 3); - printf("%lld/%lld/%lld \r", zoom, x, y); + fprintf(stderr, "%lld/%lld/%lld \r", zoom, x, y); - handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers); + handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb); } sqlite3_finalize(stmt); @@ -259,7 +307,7 @@ int main(int argc, char **argv) { char **layernames = NULL; int nlayers = 0; - decode(argv[optind], argv[optind + 1], &file_keys, &layernames, &nlayers); + decode(argv[optind], argv[optind + 1], &file_keys, &layernames, &nlayers, outdb); for (i = 0; i < nlayers; i++) { printf("%s\n", layernames[i]); From f131987f9ef2f2628bc61273012fb54488ae3d53 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 15:47:44 -0700 Subject: [PATCH 07/12] Fix my pointer confusion around the global keys per layer --- tile-join.cc | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index f73f4bb6..4aaf9397 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -120,7 +120,16 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi *file_keys = (struct pool *) realloc(*file_keys, (ll + 1) * sizeof(struct pool)); *layernames = (char **) realloc(*layernames, (ll + 1) * sizeof(char *)); - pool_init(file_keys[ll], 0); + if (*file_keys == NULL) { + perror("realloc file_keys"); + exit(EXIT_FAILURE); + } + if (*layernames == NULL) { + perror("realloc layernames"); + exit(EXIT_FAILURE); + } + + pool_init(&((*file_keys)[ll]), 0); (*layernames)[ll] = strdup(ln); *nlayers = ll + 1; } @@ -175,8 +184,8 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi continue; } - if (!is_pooled(file_keys[ll], key, type)) { - pool(file_keys[ll], strdup(key), type); + if (!is_pooled(&((*file_keys)[ll]), key, type)) { + pool(&((*file_keys)[ll]), strdup(key), type); } struct pool_val *k, *v; @@ -266,19 +275,20 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, } void usage(char **argv) { - fprintf(stderr, "Usage: %s [-f] -o new.mbtiles source.mbtiles map.csv\n", argv[0]); + fprintf(stderr, "Usage: %s [-f] [-c joins.csv] -o new.mbtiles source.mbtiles ...\n", argv[0]); exit(EXIT_FAILURE); } int main(int argc, char **argv) { char *outfile = NULL; + char *csv = NULL; int force = 0; extern int optind; extern char *optarg; int i; - while ((i = getopt(argc, argv, "fo:")) != -1) { + while ((i = getopt(argc, argv, "fo:c:")) != -1) { switch (i) { case 'o': outfile = optarg; @@ -288,12 +298,16 @@ int main(int argc, char **argv) { force = 1; break; + case 'c': + csv = optarg; + break; + default: usage(argv); } } - if (argc != optind + 2 || outfile == NULL) { + if (argc - optind < 1 || outfile == NULL) { usage(argv); } @@ -307,13 +321,20 @@ int main(int argc, char **argv) { char **layernames = NULL; int nlayers = 0; - decode(argv[optind], argv[optind + 1], &file_keys, &layernames, &nlayers, outdb); + for (i = optind; i < argc; i++) { + decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb); + } for (i = 0; i < nlayers; i++) { printf("%s\n", layernames[i]); } - mbtiles_write_metadata(outdb, outfile, layernames, 0, 0, 0, 0, 0, 0, 0, 0, &file_keys, nlayers); + struct pool *fk[nlayers]; + for (i = 0; i < nlayers; i++) { + fk[i] = &(file_keys[i]); + } + + mbtiles_write_metadata(outdb, outfile, layernames, 0, 0, 0, 0, 0, 0, 0, 0, fk, nlayers); mbtiles_close(outdb, argv); return 0; From 06ca52526d3a899bdba270dd65079226c7f63548 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 16:12:34 -0700 Subject: [PATCH 08/12] Copy the bounding box, center, zoom range metadata over --- tile-join.cc | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index 4aaf9397..dc7f0360 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -17,6 +17,13 @@ extern "C" { #include "mbtiles.h" } +struct stats { + int minzoom; + int maxzoom; + double midlat, midlon; + double minlat, minlon, maxlat, maxlon; +}; + // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp inline bool is_compressed(std::string const &data) { return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B)); @@ -237,7 +244,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } -void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -268,6 +275,33 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, sqlite3_finalize(stmt); + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'minzoom'", -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + st->minzoom = sqlite3_column_int(stmt, 0); + } + sqlite3_finalize(stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'maxzoom'", -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + st->maxzoom = sqlite3_column_int(stmt, 0); + } + sqlite3_finalize(stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'center'", -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(stmt, 0); + sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat); + } + sqlite3_finalize(stmt); + } + if (sqlite3_prepare_v2(db, "SELECT value from metadata where name = 'bounds'", -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + const unsigned char *s = sqlite3_column_text(stmt, 0); + sscanf((char *) s, "%lf,%lf,%lf,%lf", &st->minlon, &st->minlat, &st->maxlon, &st->maxlat); + } + sqlite3_finalize(stmt); + } + if (sqlite3_close(db) != SQLITE_OK) { fprintf(stderr, "%s: could not close database: %s\n", fname, sqlite3_errmsg(db)); exit(EXIT_FAILURE); @@ -316,13 +350,15 @@ int main(int argc, char **argv) { } sqlite3 *outdb = mbtiles_open(outfile, argv); + struct stats st; + memset(&st, 0, sizeof(st)); struct pool *file_keys = NULL; char **layernames = NULL; int nlayers = 0; for (i = optind; i < argc; i++) { - decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb); + decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st); } for (i = 0; i < nlayers; i++) { @@ -334,7 +370,7 @@ int main(int argc, char **argv) { fk[i] = &(file_keys[i]); } - mbtiles_write_metadata(outdb, outfile, layernames, 0, 0, 0, 0, 0, 0, 0, 0, fk, nlayers); + mbtiles_write_metadata(outdb, outfile, layernames, st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, fk, nlayers); mbtiles_close(outdb, argv); return 0; From 514da467e145a9155ec9dae4f13b3a8493c5b61a Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 16:15:09 -0700 Subject: [PATCH 09/12] Only allow one input .mbtiles because multiple really won't work --- tile-join.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index dc7f0360..d0c05f80 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -309,7 +309,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, } void usage(char **argv) { - fprintf(stderr, "Usage: %s [-f] [-c joins.csv] -o new.mbtiles source.mbtiles ...\n", argv[0]); + fprintf(stderr, "Usage: %s [-f] [-c joins.csv] -o new.mbtiles source.mbtiles\n", argv[0]); exit(EXIT_FAILURE); } @@ -341,7 +341,7 @@ int main(int argc, char **argv) { } } - if (argc - optind < 1 || outfile == NULL) { + if (argc - optind != 1 || outfile == NULL) { usage(argv); } From 9a4953409819b64dbffa7d1f3b8b5ba4342daaca Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 16:45:45 -0700 Subject: [PATCH 10/12] Parse the CSV --- tile-join.cc | 71 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index d0c05f80..bad0c785 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include #include #include "vector_tile.pb.h" @@ -88,7 +90,7 @@ static inline int compress(std::string const &input, std::string &output) { return 0; } -void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb) { +void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, std::vector &header, std::map > &mapping) { GOOGLE_PROTOBUF_VERIFY_VERSION; // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp @@ -244,7 +246,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi mbtiles_write_tile(outdb, z, x, y, compressed.data(), compressed.size()); } -void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st) { +void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, struct stats *st, std::vector &header, std::map > &mapping) { sqlite3 *db; if (sqlite3_open(fname, &db) != SQLITE_OK) { @@ -270,7 +272,7 @@ void decode(char *fname, char *map, struct pool **file_keys, char ***layernames, fprintf(stderr, "%lld/%lld/%lld \r", zoom, x, y); - handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb); + handle(std::string(s, len), zoom, x, y, file_keys, layernames, nlayers, outdb, header, mapping); } sqlite3_finalize(stmt); @@ -313,11 +315,66 @@ void usage(char **argv) { exit(EXIT_FAILURE); } +#define MAXLINE 10000 /* XXX */ + +std::vector split(char *s) { + std::vector ret; + + while (*s && *s != '\n') { + char *start = s; + int within = 0; + + for (; *s && *s != '\n'; s++) { + if (*s == '"') { + within = !within; + } + + if (*s == ',' && !within) { + break; + } + } + + std::string v = std::string(start, s - start); + ret.push_back(v); + + if (*s == ',') { + s++; + } + } + + return ret; +} + +void readcsv(char *fn, std::vector &header, std::map > &mapping) { + FILE *f = fopen(fn, "r"); + if (f == NULL) { + perror(fn); + exit(EXIT_FAILURE); + } + + char s[MAXLINE]; + if (fgets(s, MAXLINE, f)) { + header = split(s); + } + while (fgets(s, MAXLINE, f)) { + std::vector line = split(s); + + for (int i = 0; i < line.size() && i < header.size(); i++) { + mapping.insert(std::pair >(line[0], line)); + } + } + + fclose(f); +} + int main(int argc, char **argv) { char *outfile = NULL; char *csv = NULL; int force = 0; + std::vector header; + std::map > mapping; + extern int optind; extern char *optarg; int i; @@ -333,7 +390,13 @@ int main(int argc, char **argv) { break; case 'c': + if (csv != NULL) { + fprintf(stderr, "Only one -c for now\n"); + exit(EXIT_FAILURE); + } + csv = optarg; + readcsv(csv, header, mapping); break; default: @@ -358,7 +421,7 @@ int main(int argc, char **argv) { int nlayers = 0; for (i = optind; i < argc; i++) { - decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st); + decode(argv[i], csv, &file_keys, &layernames, &nlayers, outdb, &st, header, mapping); } for (i = 0; i < nlayers; i++) { From 529c9aedc9f707a85f67f9ecd367d16fd99f2c0f Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 20 Aug 2015 17:32:58 -0700 Subject: [PATCH 11/12] Join attributes from CSV into tiles --- tile-join.cc | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tile-join.cc b/tile-join.cc index bad0c785..7f0d1be4 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -19,6 +19,8 @@ extern "C" { #include "mbtiles.h" } +std::string dequote(std::string s); + struct stats { int minzoom; int maxzoom; @@ -213,6 +215,52 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi outfeature->add_tags(k->n); outfeature->add_tags(v->n); + + if (strcmp(key, header[0].c_str()) == 0) { + std::map >::iterator ii = mapping.find(std::string(value)); + + if (ii != mapping.end()) { + std::vector fields = ii->second; + + for (int i = 1; i < fields.size(); i++) { + std::string joinkey = header[i]; + std::string joinval = fields[i]; + int type = VT_STRING; + + if (joinval.size() > 0) { + if (joinval[0] == '"') { + joinval = dequote(joinval); + } else if ((joinval[0] >= '0' && joinval[0] <= '9') || joinval[0] == '-') { + type = VT_NUMBER; + } + } + + const char *sjoinkey = joinkey.c_str(); + const char *sjoinval = joinval.c_str(); + + if (!is_pooled(&((*file_keys)[ll]), sjoinkey, type)) { + pool(&((*file_keys)[ll]), strdup(sjoinkey), type); + } + + if (is_pooled(&keys, sjoinkey, VT_STRING)) { + k = pool(&keys, sjoinkey, VT_STRING); + } else { + k = pool(&keys, strdup(sjoinkey), VT_STRING); + } + + if (is_pooled(&values, sjoinval, type)) { + v = pool(&values, sjoinval, type); + } else { + v = pool(&values, strdup(sjoinval), type); + } + + outfeature->add_tags(k->n); + outfeature->add_tags(v->n); + } + } + + } + free(value); } } @@ -345,6 +393,21 @@ std::vector split(char *s) { return ret; } +std::string dequote(std::string s) { + std::string out; + int i; + for (i = 0; i < s.size(); i++) { + if (s[i] == '"') { + if (i + 1 < s.size() && s[i + 1] == '"') { + out.push_back('"'); + } + } else { + out.push_back(s[i]); + } + } + return out; +} + void readcsv(char *fn, std::vector &header, std::map > &mapping) { FILE *f = fopen(fn, "r"); if (f == NULL) { @@ -355,11 +418,19 @@ void readcsv(char *fn, std::vector &header, std::map line = split(s); + if (line.size() > 0) { + line[0] = dequote(line[0]); + } for (int i = 0; i < line.size() && i < header.size(); i++) { + // printf("putting %s\n", line[0].c_str()); mapping.insert(std::pair >(line[0], line)); } } From 88b9750959c14224877ec11c71e91b7c0be62ec3 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Fri, 21 Aug 2015 10:33:57 -0700 Subject: [PATCH 12/12] Fix a bunch of compiler warnings --- tile-join.cc | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/tile-join.cc b/tile-join.cc index 7f0d1be4..84822d78 100644 --- a/tile-join.cc +++ b/tile-join.cc @@ -1,6 +1,3 @@ -// for vasprintf() on Linux -#define _GNU_SOURCE - #include #include #include @@ -160,41 +157,46 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi } for (int t = 0; t + 1 < feat.tags_size(); t += 2) { - if (feat.tags(t) >= layer.keys_size() || feat.tags(t + 1) >= layer.values_size()) { - printf("out of range: %d=%d\n", feat.tags(t), feat.tags(t + 1)); - continue; - } - const char *key = layer.keys(feat.tags(t)).c_str(); mapnik::vector::tile_value const &val = layer.values(feat.tags(t + 1)); char *value; - int type; + int type = -1; if (val.has_string_value()) { value = strdup(val.string_value().c_str()); type = VT_STRING; } else if (val.has_int_value()) { - asprintf(&value, "%lld", val.int_value()); - type = VT_NUMBER; + if (asprintf(&value, "%lld", (long long) val.int_value()) >= 0) { + type = VT_NUMBER; + } } else if (val.has_double_value()) { - asprintf(&value, "%g", val.double_value()); - type = VT_NUMBER; + if (asprintf(&value, "%g", val.double_value()) >= 0) { + type = VT_NUMBER; + } } else if (val.has_float_value()) { - asprintf(&value, "%g", val.float_value()); - type = VT_NUMBER; + if (asprintf(&value, "%g", val.float_value()) >= 0) { + type = VT_NUMBER; + } } else if (val.has_bool_value()) { - asprintf(&value, "%s", val.bool_value() ? "true" : "false"); - type = VT_BOOLEAN; + if (asprintf(&value, "%s", val.bool_value() ? "true" : "false") >= 0) { + type = VT_BOOLEAN; + } } else if (val.has_sint_value()) { - asprintf(&value, "%lld", val.sint_value()); - type = VT_NUMBER; + if (asprintf(&value, "%lld", (long long) val.sint_value()) >= 0) { + type = VT_NUMBER; + } } else if (val.has_uint_value()) { - asprintf(&value, "%llu", val.uint_value()); - type = VT_NUMBER; + if (asprintf(&value, "%llu", (long long) val.uint_value()) >= 0) { + type = VT_NUMBER; + } } else { continue; } + if (type < 0) { + continue; + } + if (!is_pooled(&((*file_keys)[ll]), key, type)) { pool(&((*file_keys)[ll]), strdup(key), type); } @@ -222,7 +224,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi if (ii != mapping.end()) { std::vector fields = ii->second; - for (int i = 1; i < fields.size(); i++) { + for (unsigned i = 1; i < fields.size(); i++) { std::string joinkey = header[i]; std::string joinval = fields[i]; int type = VT_STRING; @@ -395,7 +397,7 @@ std::vector split(char *s) { std::string dequote(std::string s) { std::string out; - int i; + unsigned i; for (i = 0; i < s.size(); i++) { if (s[i] == '"') { if (i + 1 < s.size() && s[i + 1] == '"') { @@ -419,7 +421,7 @@ void readcsv(char *fn, std::vector &header, std::map &header, std::map >(line[0], line)); }