diff --git a/Makefile b/Makefile index 5ca63688..b499578d 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ tippecanoe-enumerate: enumerate.o tippecanoe-decode: decode.o projection.o mvt.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o +tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread unit: unit.o text.o @@ -116,7 +116,7 @@ parallel-test: raw-tiles-test: ./tippecanoe -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC - diff -x '.*' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare + diff -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare rm -rf tests/raw-tiles/raw-tiles decode-test: @@ -162,8 +162,11 @@ join-test: ./tippecanoe-decode tests/join-population/just-macarthur.mbtiles > tests/join-population/just-macarthur.mbtiles.json.check ./tippecanoe-decode tests/join-population/no-macarthur.mbtiles > tests/join-population/no-macarthur.mbtiles.json.check cmp tests/join-population/just-macarthur.mbtiles.json.check tests/join-population/just-macarthur.mbtiles.json - cmp tests/join-population/no-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json + cmp tests/join-population/no-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json + ./tile-join -pC -e tests/join-population/raw-merged-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles + diff -rq tests/join-population/raw-merged-folder tests/join-population/raw-merged-folder-compare rm tests/join-population/tabblock_06001420.mbtiles tests/join-population/joined.mbtiles tests/join-population/joined-i.mbtiles tests/join-population/joined.mbtiles.json.check tests/join-population/joined-i.mbtiles.json.check tests/join-population/macarthur.mbtiles tests/join-population/merged.mbtiles tests/join-population/merged.mbtiles.json.check tests/join-population/macarthur2.mbtiles tests/join-population/windows.mbtiles tests/join-population/windows.mbtiles.json.check tests/join-population/just-macarthur.mbtiles tests/join-population/no-macarthur.mbtiles tests/join-population/just-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json.check + rm -rf tests/join-population/raw-merged-folder # Use this target to regenerate the standards that the tests are compared against # after making a change that legitimately changes their output diff --git a/README.md b/README.md index 1debe763..07332f5b 100644 --- a/README.md +++ b/README.md @@ -402,12 +402,14 @@ the same tiles, the layers or tiles are merged. The options are: - * `-o` *out.mbtiles*: Write the new tiles to the specified .mbtiles file - * `-f`: Remove *out.mbtiles* if it already exists + * `-o` *out.mbtiles*: Write the new tiles to the specified .mbtiles file. + * `-e` *directory*: Write the new tiles to the specified directory instead of to an mbtiles file. + * `-f`: Remove *out.mbtiles* if it already exists. * `-c` *match*`.csv`: Use *match*`.csv` as the source for new attributes to join to the features. The first line of the file should be the key names; the other lines are values. The first column is the one to match against the existing features; the other columns are the new data to add. * `-x` *key*: Remove attributes of type *key* from the output. You can use this to remove the field you are matching against if you no longer need it after joining, or to remove any other attributes you don't want. * `-i`: Only include features that matched the CSV. * `-pk`: Don't skip tiles larger than 500K. + * `-pC`: Don't compress the PBF vector tile data. * `-l` *layer*: Include the named layer in the output. You can specify multiple `-l` options to keep multiple layers. If you don't specify, they will all be retained. * `-L` *layer*: Remove the named layer from the output. You can specify multiple `-L` options to remove multiple layers. * `-A` *attribution*: Set the attribution string. diff --git a/tests/join-population/raw-merged-folder-compare/10/164/395.pbf b/tests/join-population/raw-merged-folder-compare/10/164/395.pbf new file mode 100644 index 00000000..3ae74128 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/10/164/395.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/10/164/396.pbf b/tests/join-population/raw-merged-folder-compare/10/164/396.pbf new file mode 100644 index 00000000..16682cd0 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/10/164/396.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/11/328/790.pbf b/tests/join-population/raw-merged-folder-compare/11/328/790.pbf new file mode 100644 index 00000000..667bd576 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/11/328/790.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/11/328/791.pbf b/tests/join-population/raw-merged-folder-compare/11/328/791.pbf new file mode 100644 index 00000000..1aebd6a9 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/11/328/791.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/11/329/791.pbf b/tests/join-population/raw-merged-folder-compare/11/329/791.pbf new file mode 100644 index 00000000..ae4820ef Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/11/329/791.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/12/656/1581.pbf b/tests/join-population/raw-merged-folder-compare/12/656/1581.pbf new file mode 100644 index 00000000..3f56006f Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/12/656/1581.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/3/1/3.pbf b/tests/join-population/raw-merged-folder-compare/3/1/3.pbf new file mode 100644 index 00000000..9eea2fc6 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/3/1/3.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/4/2/6.pbf b/tests/join-population/raw-merged-folder-compare/4/2/6.pbf new file mode 100644 index 00000000..4478ef57 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/4/2/6.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/5/5/12.pbf b/tests/join-population/raw-merged-folder-compare/5/5/12.pbf new file mode 100644 index 00000000..fb0fa91a Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/5/5/12.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/6/10/24.pbf b/tests/join-population/raw-merged-folder-compare/6/10/24.pbf new file mode 100644 index 00000000..32cef6eb Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/6/10/24.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/7/20/49.pbf b/tests/join-population/raw-merged-folder-compare/7/20/49.pbf new file mode 100644 index 00000000..bdfa9dbb Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/7/20/49.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/8/40/98.pbf b/tests/join-population/raw-merged-folder-compare/8/40/98.pbf new file mode 100644 index 00000000..3177a515 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/8/40/98.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/8/41/98.pbf b/tests/join-population/raw-merged-folder-compare/8/41/98.pbf new file mode 100644 index 00000000..51964cd8 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/8/41/98.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/8/41/99.pbf b/tests/join-population/raw-merged-folder-compare/8/41/99.pbf new file mode 100644 index 00000000..d64337b4 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/8/41/99.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/9/81/197.pbf b/tests/join-population/raw-merged-folder-compare/9/81/197.pbf new file mode 100644 index 00000000..a449a0e1 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/9/81/197.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/9/82/197.pbf b/tests/join-population/raw-merged-folder-compare/9/82/197.pbf new file mode 100644 index 00000000..aa12d7c4 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/9/82/197.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/9/82/198.pbf b/tests/join-population/raw-merged-folder-compare/9/82/198.pbf new file mode 100644 index 00000000..b5e51ae6 Binary files /dev/null and b/tests/join-population/raw-merged-folder-compare/9/82/198.pbf differ diff --git a/tests/join-population/raw-merged-folder-compare/metadata.json b/tests/join-population/raw-merged-folder-compare/metadata.json new file mode 100644 index 00000000..6ca20d8a --- /dev/null +++ b/tests/join-population/raw-merged-folder-compare/metadata.json @@ -0,0 +1,12 @@ +{ + "name": "tests/join-population/macarthur.mbtiles + tests/join-population/macarthur2.mbtiles + tests/join-population/tabblock_06001420.mbtiles", + "description": "tests/join-population/tabblock_06001420.mbtiles", + "version": "2", + "minzoom": "0", + "maxzoom": "12", + "center": "-122.299805,37.892187,12", + "bounds": "-122.343750,37.695438,-122.104097,37.926868", + "type": "overlay", + "format": "pbf", + "json": "{\"vector_layers\": [ { \"id\": \"macarthur\", \"description\": \"\", \"minzoom\": 5, \"maxzoom\": 11, \"fields\": {\"FULLNAME\": \"String\", \"LINEARID\": \"String\", \"MTFCC\": \"String\", \"RTTYP\": \"String\"} }, { \"id\": \"tabblock_06001420\", \"description\": \"\", \"minzoom\": 3, \"maxzoom\": 12, \"fields\": {\"ALAND10\": \"Number\", \"AWATER10\": \"Number\", \"BLOCKCE10\": \"String\", \"COUNTYFP10\": \"String\", \"FUNCSTAT10\": \"String\", \"GEOID10\": \"String\", \"INTPTLAT10\": \"String\", \"INTPTLON10\": \"String\", \"MTFCC10\": \"String\", \"NAME10\": \"String\", \"STATEFP10\": \"String\", \"TRACTCE10\": \"String\", \"UACE10\": \"String\", \"UATYP10\": \"String\", \"UR10\": \"String\"} } ] }" +} diff --git a/tile-join.cpp b/tile-join.cpp index f06107c5..78612f71 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -16,10 +16,12 @@ #include "pool.hpp" #include "mbtiles.hpp" #include "geometry.hpp" +#include "dirtiles.hpp" std::string dequote(std::string s); bool pk = false; +bool pC = false; size_t CPUS; struct stats { @@ -405,7 +407,12 @@ void *join_worker(void *v) { if (anything) { std::string pbf = tile.encode(); std::string compressed; - compress(pbf, compressed); + + if (!pC) { + compress(pbf, compressed); + } else { + compressed = pbf; + } if (!pk && compressed.size() > 500000) { fprintf(stderr, "Tile %lld/%lld/%lld size is %lld, >500000. Skipping this tile\n.", ai->first.z, ai->first.x, ai->first.y, (long long) compressed.size()); @@ -418,7 +425,7 @@ void *join_worker(void *v) { return NULL; } -void handle_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::set &keep_layers, std::set &remove_layers) { +void handle_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::set &keep_layers, std::set &remove_layers) { pthread_t pthreads[CPUS]; std::vector args; @@ -462,12 +469,16 @@ void handle_tasks(std::map> &tasks, std::vectorfirst.z, ai->first.x, ai->first.y, ai->second.data(), ai->second.size()); + if (outdb != NULL) { + mbtiles_write_tile(outdb, ai->first.z, ai->first.x, ai->first.y, ai->second.data(), ai->second.size()); + } else if (outdir != NULL) { + dir_write_tile(outdir, ai->first.z, ai->first.x, ai->first.y, ai->second); + } } } } -void decode(struct reader *readers, char *map, std::map &layermap, sqlite3 *outdb, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name) { +void decode(struct reader *readers, char *map, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name) { std::vector> layermaps; for (size_t i = 0; i < CPUS; i++) { layermaps.push_back(std::map()); @@ -512,7 +523,7 @@ void decode(struct reader *readers, char *map, std::mapzoom != r->zoom || readers->x != r->x || readers->y != r->y) { if (tasks.size() > 100 * CPUS) { - handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched, keep_layers, remove_layers); + handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers); tasks.clear(); } } @@ -548,7 +559,7 @@ void decode(struct reader *readers, char *map, std::mapminlat = min(minlat, st->minlat); st->maxlat = max(maxlat, st->maxlat); - handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched, keep_layers, remove_layers); + handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers); layermap = merge_layermaps(layermaps); struct reader *next; @@ -622,7 +633,7 @@ void decode(struct reader *readers, char *map, std::map &header, std::map