Merge pull request #424 from joykuotw/master

Add tile-join directory support

Makefile

@@ -46,7 +46,7 @@ C = $(wildcard *.c) $(wildcard *.cpp)
 INCLUDES = -I/usr/local/include -I.
 LIBS = -L/usr/local/lib
 
-tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o rawtiles.o
+tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 tippecanoe-enumerate: enumerate.o
@@ -55,7 +55,7 @@ tippecanoe-enumerate: enumerate.o
 tippecanoe-decode: decode.o projection.o mvt.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
-tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o
+tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 unit: unit.o text.o
@@ -116,7 +116,7 @@ parallel-test:
 
 raw-tiles-test:	
 	./tippecanoe -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC
-	diff -x '.*' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare
+	diff -x '*.DS_Store' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare
 	rm -rf tests/raw-tiles/raw-tiles
 
 decode-test:
@@ -163,7 +163,31 @@ join-test:
 	./tippecanoe-decode tests/join-population/no-macarthur.mbtiles > tests/join-population/no-macarthur.mbtiles.json.check
 	cmp tests/join-population/just-macarthur.mbtiles.json.check tests/join-population/just-macarthur.mbtiles.json
 	cmp tests/join-population/no-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json	
-	rm tests/join-population/tabblock_06001420.mbtiles tests/join-population/joined.mbtiles tests/join-population/joined-i.mbtiles tests/join-population/joined.mbtiles.json.check tests/join-population/joined-i.mbtiles.json.check tests/join-population/macarthur.mbtiles tests/join-population/merged.mbtiles tests/join-population/merged.mbtiles.json.check tests/join-population/macarthur2.mbtiles tests/join-population/windows.mbtiles tests/join-population/windows.mbtiles.json.check tests/join-population/just-macarthur.mbtiles tests/join-population/no-macarthur.mbtiles tests/join-population/just-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json.check
+	./tile-join -pC -e tests/join-population/raw-merged-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
+	diff -x '*.DS_Store' -rq tests/join-population/raw-merged-folder tests/join-population/raw-merged-folder-compare
+	./tippecanoe -z12 -e tests/join-population/tabblock_06001420-folder tests/join-population/tabblock_06001420.json
+	./tippecanoe -Z5 -z10 -e tests/join-population/macarthur-folder -l macarthur tests/join-population/macarthur.json
+	./tippecanoe -d10 -D10 -Z9 -z11 -e tests/join-population/macarthur2-folder -l macarthur tests/join-population/macarthur2.json
+	./tile-join -f -o tests/join-population/merged-folder.mbtiles tests/join-population/tabblock_06001420-folder tests/join-population/macarthur-folder tests/join-population/macarthur2-folder
+	./tippecanoe-decode tests/join-population/merged-folder.mbtiles > tests/join-population/merged-folder.mbtiles.json.check
+	cmp tests/join-population/merged-folder.mbtiles.json.check tests/join-population/merged-folder.mbtiles.json
+	./tile-join -n "merged name" -N "merged description" -e tests/join-population/merged-mbtiles-to-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
+	./tile-join -n "merged name" -N "merged description" -e tests/join-population/merged-folders-to-folder tests/join-population/tabblock_06001420-folder tests/join-population/macarthur-folder tests/join-population/macarthur2-folder
+	diff -x '*.DS_Store' -rq tests/join-population/merged-mbtiles-to-folder tests/join-population/merged-folders-to-folder
+	./tile-join -f -c tests/join-population/windows.csv -o tests/join-population/windows-merged.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
+	./tile-join -c tests/join-population/windows.csv -e tests/join-population/windows-merged-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
+	./tile-join -f -o tests/join-population/windows-merged2.mbtiles tests/join-population/windows-merged-folder
+	./tippecanoe-decode tests/join-population/windows-merged.mbtiles > tests/join-population/windows-merged.mbtiles.json.check
+	./tippecanoe-decode tests/join-population/windows-merged2.mbtiles > tests/join-population/windows-merged2.mbtiles.json.check
+	cmp tests/join-population/windows-merged.mbtiles.json.check tests/join-population/windows-merged2.mbtiles.json.check
+	./tile-join -f -o tests/join-population/macarthur-and-macarthur2-merged.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
+	./tile-join -e tests/join-population/macarthur-and-macarthur2-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
+	./tile-join -f -o tests/join-population/macarthur-and-macarthur2-merged2.mbtiles tests/join-population/macarthur-and-macarthur2-folder
+	./tippecanoe-decode tests/join-population/macarthur-and-macarthur2-merged.mbtiles > tests/join-population/macarthur-and-macarthur2-merged.mbtiles.json.check
+	./tippecanoe-decode tests/join-population/macarthur-and-macarthur2-merged2.mbtiles > tests/join-population/macarthur-and-macarthur2-merged2.mbtiles.json.check
+	cmp tests/join-population/macarthur-and-macarthur2-merged.mbtiles.json.check tests/join-population/macarthur-and-macarthur2-merged2.mbtiles.json.check
+	rm tests/join-population/tabblock_06001420.mbtiles tests/join-population/joined.mbtiles tests/join-population/joined-i.mbtiles tests/join-population/joined.mbtiles.json.check tests/join-population/joined-i.mbtiles.json.check tests/join-population/macarthur.mbtiles tests/join-population/merged.mbtiles tests/join-population/merged.mbtiles.json.check  tests/join-population/merged-folder.mbtiles tests/join-population/macarthur2.mbtiles tests/join-population/windows.mbtiles tests/join-population/windows-merged.mbtiles tests/join-population/windows-merged2.mbtiles tests/join-population/windows.mbtiles.json.check tests/join-population/just-macarthur.mbtiles tests/join-population/no-macarthur.mbtiles tests/join-population/just-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json.check tests/join-population/merged-folder.mbtiles.json.check tests/join-population/windows-merged.mbtiles.json.check tests/join-population/windows-merged2.mbtiles.json.check tests/join-population/macarthur-and-macarthur2-merged.mbtiles tests/join-population/macarthur-and-macarthur2-merged2.mbtiles tests/join-population/macarthur-and-macarthur2-merged.mbtiles.json.check tests/join-population/macarthur-and-macarthur2-merged2.mbtiles.json.check
+	rm -rf tests/join-population/raw-merged-folder tests/join-population/tabblock_06001420-folder tests/join-population/macarthur-folder tests/join-population/macarthur2-folder tests/join-population/merged-mbtiles-to-folder tests/join-population/merged-folders-to-folder tests/join-population/windows-merged-folder tests/join-population/macarthur-and-macarthur2-folder
 
 # Use this target to regenerate the standards that the tests are compared against
 # after making a change that legitimately changes their output

README.md

@@ -392,22 +392,29 @@ The name is [a joking reference](http://en.wikipedia.org/wiki/Tippecanoe_and_Tyl
 tile-join
 =========
 
-Tile-join is a tool for joining new attributes from a CSV file to features that
+Tile-join is a tool for joining new attributes from a CSV file to features
-have already been tiled with tippecanoe. It reads the tiles from an existing .mbtiles
+that have already been tiled with tippecanoe. It reads the tiles from an 
-file, matches them against the records of the CSV, and writes out a new tileset.
+existing .mbtiles file or a directory of tiles, matches them against the
+records of the CSV, and writes out a new tileset.
 
-If you specify multiple source mbtiles files, they are all read and their combined
+If you specify multiple source mbtiles files or source directories of tiles,
-contents are written to the new mbtiles output. If they define the same layers or
+all the sources are read and their combined contents are written to the new 
-the same tiles, the layers or tiles are merged.
+mbtiles output. If they define the same layers or the same tiles, the layers
+or tiles are merged.
+
+You can use the `-e` flag to output a directory of tiles rather than a
+.mbtiles file.
 
 The options are:
 
- * `-o` *out.mbtiles*: Write the new tiles to the specified .mbtiles file
+ * `-o` *out.mbtiles*: Write the new tiles to the specified .mbtiles file.
- * `-f`: Remove *out.mbtiles* if it already exists
+ * `-e` *directory*: Write the new tiles to the specified directory instead of to an mbtiles file.
+ * `-f`: Remove *out.mbtiles* if it already exists.
  * `-c` *match*`.csv`: Use *match*`.csv` as the source for new attributes to join to the features. The first line of the file should be the key names; the other lines are values. The first column is the one to match against the existing features; the other columns are the new data to add.
  * `-x` *key*: Remove attributes of type *key* from the output. You can use this to remove the field you are matching against if you no longer need it after joining, or to remove any other attributes you don't want.
  * `-i`: Only include features that matched the CSV.
  * `-pk`: Don't skip tiles larger than 500K.
+ * `-pC`: Don't compress the PBF vector tile data.
  * `-l` *layer*: Include the named layer in the output. You can specify multiple `-l` options to keep multiple layers. If you don't specify, they will all be retained.
  * `-L` *layer*: Remove the named layer from the output. You can specify multiple `-L` options to remove multiple layers.
  * `-A` *attribution*: Set the attribution string.

dirtiles.cpp

@@ -1,10 +1,20 @@
 #include <iostream>
 #include <fstream>
+#include <sstream>
 #include <string>
 #include <sys/stat.h>
-#include "rawtiles.hpp"
+#include "dirtiles.hpp"
 
-void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) {
+std::string dir_read_tile(std::string pbfPath) {
+	std::ifstream pbfFile(pbfPath, std::ios::in | std::ios::binary);
+	std::ostringstream contents;
+	contents << pbfFile.rdbuf();
+	pbfFile.close();
+
+	return (contents.str());
+}
+
+void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) {
 	mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO);
 	std::string curdir(outdir);
 	std::string slash("/");

dirtiles.hpp

@@ -0,0 +1,3 @@
+std::string dir_read_tile(std::string pbfPath);
+
+void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);

rawtiles.hpp

@@ -1 +0,0 @@
-void write_raw_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);

tests/join-population/raw-merged-folder-compare/metadata.json

@@ -0,0 +1,12 @@
+{
+    "name": "tests/join-population/macarthur.mbtiles + tests/join-population/macarthur2.mbtiles + tests/join-population/tabblock_06001420.mbtiles",
+    "description": "tests/join-population/tabblock_06001420.mbtiles",
+    "version": "2",
+    "minzoom": "0",
+    "maxzoom": "12",
+    "center": "-122.299805,37.892187,12",
+    "bounds": "-122.343750,37.695438,-122.104097,37.926868",
+    "type": "overlay",
+    "format": "pbf",
+    "json": "{\"vector_layers\": [ { \"id\": \"macarthur\", \"description\": \"\", \"minzoom\": 5, \"maxzoom\": 11, \"fields\": {\"FULLNAME\": \"String\", \"LINEARID\": \"String\", \"MTFCC\": \"String\", \"RTTYP\": \"String\"} }, { \"id\": \"tabblock_06001420\", \"description\": \"\", \"minzoom\": 3, \"maxzoom\": 12, \"fields\": {\"ALAND10\": \"Number\", \"AWATER10\": \"Number\", \"BLOCKCE10\": \"String\", \"COUNTYFP10\": \"String\", \"FUNCSTAT10\": \"String\", \"GEOID10\": \"String\", \"INTPTLAT10\": \"String\", \"INTPTLON10\": \"String\", \"MTFCC10\": \"String\", \"NAME10\": \"String\", \"STATEFP10\": \"String\", \"TRACTCE10\": \"String\", \"UACE10\": \"String\", \"UATYP10\": \"String\", \"UR10\": \"String\"} } ] }"
+}

tile-join.cpp

@@ -1,3 +1,7 @@
+#define _DEFAULT_SOURCE
+#include <dirent.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -16,10 +20,20 @@
 #include "pool.hpp"
 #include "mbtiles.hpp"
 #include "geometry.hpp"
+#include "dirtiles.hpp"
+#include <fstream>
+#include <sstream>
+#include <algorithm>
+#include <functional>
+
+extern "C" {
+#include "jsonpull/jsonpull.h"
+}
 
 std::string dequote(std::string s);
 
 bool pk = false;
+bool pC = false;
 size_t CPUS;
 
 struct stats {
@@ -260,8 +274,12 @@ struct reader {
 	long long x;
 	long long sorty;
 	long long y;
+	int pbf_count;
+	int z_flag;
 
 	std::string data;
+	std::vector<std::string> pbf_path;
+	std::vector<std::string> large_zoom;
 
 	sqlite3 *db;
 	sqlite3_stmt *stmt;
@@ -297,7 +315,166 @@ struct reader {
 	}
 };
 
+std::vector<std::string> split_slash(std::string pbf_path) {
+	std::vector<std::string> path_parts;
+	std::string path(pbf_path);
+	std::istringstream iss(path);
+	std::string token;
+
+	while (std::getline(iss, token, '/')) {
+		path_parts.push_back(token);
+	}
+
+	return path_parts;
+}
+
+int filter(const struct dirent *dir) {
+	if (strcmp(dir->d_name, ".") == 0 || strcmp(dir->d_name, "..") == 0 || strcmp(dir->d_name, ".DS_Store") == 0 || strcmp(dir->d_name, "metadata.json") == 0) {
+		return 0;
+	} else {
+		return 1;
+	}
+}
+
+// Recursively walk through a specified directory and its subdirectories,
+// using alphasort function and integer variable zoom_range to handle input in numerical order.
+// Store the path of all pbf files in a pbf_path vector member of reader struct,
+// with the help of a large_zoom vector and two integer members pbf_count and z_flag
+// to ensure the tiles order in pbf_path to be the same as in mbtiles.
+struct reader *read_dir(struct reader *readers, const char *name, int level, int zoom_range) {
+	struct dirent **namelist;
+	struct stat buf;
+	std::string path;
+	int i = 0;
+	int n = scandir(name, &namelist, filter, alphasort);
+	std::vector<std::string> path_parts1, path_parts2;
+	readers->pbf_count = 0;
+
+	if (n > 0) {
+		while (i < n) {
+			path = std::string(name) + "/" + std::string(namelist[i]->d_name);
+
+			if (stat(path.c_str(), &buf) == 0 && S_ISDIR(buf.st_mode)) {
+				if (level == 0) {
+					if (std::stoi(namelist[i]->d_name) <= 9) {
+						zoom_range = 0;
+					} else {
+						zoom_range = 1;
+					}
+
+					if (readers->pbf_count > 0) {
+						if (readers->z_flag == 0) {
+							std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater<std::string>());
+						} else {
+							std::sort(readers->large_zoom.end() - (readers->pbf_count + 1), readers->large_zoom.end(), std::greater<std::string>());
+						}
+						readers->pbf_count = 0;
+					}
+				}
+
+				if (level == 1 && readers->pbf_count > 0) {
+					if (zoom_range == 0) {
+						std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater<std::string>());
+					} else {
+						std::sort(readers->large_zoom.end() - (readers->pbf_count + 1), readers->large_zoom.end(), std::greater<std::string>());
+					}
+					readers->pbf_count = 0;
+				}
+
+				read_dir(readers, path.c_str(), level + 1, zoom_range);
+			} else {
+				if (level == 0) {
+					fprintf(stderr, "ERROR: Directory structure in '%s' should be zoom/x/y\n", name);
+					exit(EXIT_FAILURE);
+				}
+
+				if (level == 1) {
+					fprintf(stderr, "ERROR: Directory structure in '%s' should be zoom/x/y\n", split_slash(name)[0].c_str());
+					exit(EXIT_FAILURE);
+				}
+
+				if (zoom_range == 0) {
+					readers->pbf_path.push_back(path);
+
+					if (readers->pbf_path.size() > 1) {
+						path_parts1 = split_slash(readers->pbf_path[readers->pbf_path.size() - 1]);
+						path_parts2 = split_slash(readers->pbf_path[readers->pbf_path.size() - 2]);
+						int p1 = path_parts1.size();
+						int p2 = path_parts2.size();
+
+						if (std::stoll(path_parts1[p1 - 3]) == std::stoll(path_parts2[p2 - 3]) && std::stoll(path_parts1[p1 - 2]) == std::stoll(path_parts2[p2 - 2])) {
+							readers->z_flag = 0;
+							readers->pbf_count++;
+						}
+
+						path_parts1.clear();
+						path_parts2.clear();
+					}
+				} else {
+					readers->large_zoom.push_back(path);
+
+					if (readers->large_zoom.size() > 1) {
+						path_parts1 = split_slash(readers->large_zoom[readers->large_zoom.size() - 1]);
+						path_parts2 = split_slash(readers->large_zoom[readers->large_zoom.size() - 2]);
+						int p1 = path_parts1.size();
+						int p2 = path_parts2.size();
+
+						if (std::stoll(path_parts1[p1 - 3]) == std::stoll(path_parts2[p2 - 3]) && std::stoll(path_parts1[p1 - 2]) == std::stoll(path_parts2[p2 - 2])) {
+							readers->z_flag = 1;
+							readers->pbf_count++;
+						}
+
+						path_parts1.clear();
+						path_parts2.clear();
+					}
+				}
+			}
+
+			free(namelist[i]);
+			i++;
+		}
+
+		if (level == 0) {
+			if (readers->pbf_count > 0) {
+				std::sort(readers->pbf_path.end() - (readers->pbf_count + 1), readers->pbf_path.end(), std::greater<std::string>());
+			}
+
+			readers->pbf_path.insert(std::end(readers->pbf_path), std::begin(readers->large_zoom), std::end(readers->large_zoom));
+		}
+
+		free(namelist);
+	} else if (n == 0) {
+		fprintf(stderr, "ERROR: Empty directory '%s'\n", name);
+		exit(EXIT_FAILURE);
+	} else {
+		perror("scandir");
+	}
+
+	return readers;
+}
+
 struct reader *begin_reading(char *fname) {
+	DIR *dir;
+	struct reader *r = new reader;
+	if ((dir = opendir(fname)) != NULL) {
+		r = read_dir(r, fname, 0, 0);
+
+		std::vector<std::string> path_parts;
+		path_parts = split_slash(r->pbf_path[0]);
+		int p = path_parts.size();
+
+		r->db = NULL;
+		r->stmt = NULL;
+		r->next = NULL;
+		r->pbf_count = 0;
+		r->zoom = std::stoll(path_parts[p - 3]);
+		r->x = std::stoll(path_parts[p - 2]);
+		r->y = std::stoll(path_parts[p - 1].substr(0, path_parts[p - 1].find_last_of(".")));
+		r->sorty = (1LL << r->zoom) - 1 - r->y;
+		r->data = dir_read_tile(r->pbf_path[0]);
+		path_parts.clear();
+		closedir(dir);
+	} else {
 		sqlite3 *db;
 
 		if (sqlite3_open(fname, &db) != SQLITE_OK) {
@@ -307,12 +484,12 @@ struct reader *begin_reading(char *fname) {
 
 		const char *sql = "SELECT zoom_level, tile_column, tile_row, tile_data from tiles order by zoom_level, tile_column, tile_row;";
 		sqlite3_stmt *stmt;
+
 		if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
 			fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
 			exit(EXIT_FAILURE);
 		}
 
-	struct reader *r = new reader;
 		r->db = db;
 		r->stmt = stmt;
 		r->next = NULL;
@@ -330,6 +507,7 @@ struct reader *begin_reading(char *fname) {
 		} else {
 			r->zoom = 32;
 		}
+	}
 
 	return r;
 }
@@ -405,7 +583,12 @@ void *join_worker(void *v) {
 		if (anything) {
 			std::string pbf = tile.encode();
 			std::string compressed;
+
+			if (!pC) {
 				compress(pbf, compressed);
+			} else {
+				compressed = pbf;
+			}
 
 			if (!pk && compressed.size() > 500000) {
 				fprintf(stderr, "Tile %lld/%lld/%lld size is %lld, >500000. Skipping this tile\n.", ai->first.z, ai->first.x, ai->first.y, (long long) compressed.size());
@@ -418,7 +601,7 @@ void *join_worker(void *v) {
 	return NULL;
 }
 
-void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers) {
+void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, const char *outdir, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers) {
 	pthread_t pthreads[CPUS];
 	std::vector<arg> args;
 
@@ -462,19 +645,22 @@ void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<st
 		}
 
 		for (auto ai = args[i].outputs.begin(); ai != args[i].outputs.end(); ++ai) {
+			if (outdb != NULL) {
 				mbtiles_write_tile(outdb, ai->first.z, ai->first.x, ai->first.y, ai->second.data(), ai->second.size());
+			} else if (outdir != NULL) {
+				dir_write_tile(outdir, ai->first.z, ai->first.x, ai->first.y, ai->second);
+			}
 		}
 	}
 }
 
-void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name) {
+void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name) {
 	std::vector<std::map<std::string, layermap_entry>> layermaps;
 	for (size_t i = 0; i < CPUS; i++) {
 		layermaps.push_back(std::map<std::string, layermap_entry>());
 	}
 
 	std::map<zxy, std::vector<std::string>> tasks;
-
 	double minlat = INT_MAX;
 	double minlon = INT_MAX;
 	double maxlat = INT_MIN;
@@ -485,7 +671,6 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 		reader *r = readers;
 		readers = readers->next;
 		r->next = NULL;
-
 		if (r->zoom != zoom_for_bbox) {
 			// Only use highest zoom for bbox calculation
 			// to avoid z0 always covering the world
@@ -512,17 +697,17 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 
 		if (readers == NULL || readers->zoom != r->zoom || readers->x != r->x || readers->y != r->y) {
 			if (tasks.size() > 100 * CPUS) {
-				handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
+				handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
 				tasks.clear();
 			}
 		}
 
+		if (r->db != NULL) {
 			if (sqlite3_step(r->stmt) == SQLITE_ROW) {
 				r->zoom = sqlite3_column_int(r->stmt, 0);
 				r->x = sqlite3_column_int(r->stmt, 1);
 				r->sorty = sqlite3_column_int(r->stmt, 2);
 				r->y = (1LL << r->zoom) - 1 - r->sorty;
-
 				const char *data = (const char *) sqlite3_column_blob(r->stmt, 3);
 				size_t len = sqlite3_column_bytes(r->stmt, 3);
 
@@ -530,6 +715,23 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 			} else {
 				r->zoom = 32;
 			}
+		} else {
+			r->pbf_count++;
+
+			if (r->pbf_count != static_cast<int>(r->pbf_path.size())) {
+				std::vector<std::string> path_parts;
+				path_parts = split_slash(r->pbf_path[r->pbf_count]);
+				int p = path_parts.size();
+				r->zoom = std::stoll(path_parts[p - 3]);
+				r->x = std::stoll(path_parts[p - 2]);
+				r->y = std::stoll(path_parts[p - 1].substr(0, path_parts[p - 1].find_last_of(".")));
+				r->sorty = (1LL << r->zoom) - 1 - r->y;
+				r->data = dir_read_tile(r->pbf_path[r->pbf_count]);
+				path_parts.clear();
+			} else {
+				r->zoom = 32;
+			}
+		}
 
 		struct reader **rr;
 
@@ -548,12 +750,14 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 	st->minlat = min(minlat, st->minlat);
 	st->maxlat = max(maxlat, st->maxlat);
 
-	handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
+	handle_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, ifmatched, keep_layers, remove_layers);
 	layermap = merge_layermaps(layermaps);
 
 	struct reader *next;
 	for (struct reader *r = readers; r != NULL; r = next) {
 		next = r->next;
+
+		if (r->db != NULL) {
 			sqlite3_finalize(r->stmt);
 
 			if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'minzoom'", -1, &r->stmt, NULL) == SQLITE_OK) {
@@ -617,12 +821,88 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 				exit(EXIT_FAILURE);
 			}
 
+		} else {
+			std::vector<std::string> path_parts;
+			path_parts = split_slash(r->pbf_path[0]);
+			std::string metadata_path = path_parts[0];
+
+			for (int i = 1; i < static_cast<int>(path_parts.size()) - 3; i++) {
+				metadata_path = metadata_path + "/" + path_parts[i];
+			}
+
+			metadata_path += "/metadata.json";
+
+			path_parts.clear();
+			FILE *f = fopen(metadata_path.c_str(), "r");
+
+			if (f == NULL) {
+				perror(metadata_path.c_str());
+				exit(EXIT_FAILURE);
+			}
+
+			json_pull *jp = json_begin_file(f);
+			json_object *j, *k;
+
+			while ((j = json_read(jp)) != NULL) {
+				if (j->type == JSON_HASH) {
+					if ((k = json_hash_get(j, "minzoom")) != NULL) {
+						const std::string minzoom_tmp = k->string;
+						int minzoom = std::stoi(minzoom_tmp);
+						st->minzoom = min(st->minzoom, minzoom);
+					}
+
+					if ((k = json_hash_get(j, "maxzoom")) != NULL) {
+						const std::string maxzoom_tmp = k->string;
+						int maxzoom = std::stoi(maxzoom_tmp);
+						st->maxzoom = max(st->maxzoom, maxzoom);
+					}
+
+					if ((k = json_hash_get(j, "center")) != NULL) {
+						const std::string center = k->string;
+						const unsigned char *s = (const unsigned char *) center.c_str();
+						sscanf((char *) s, "%lf,%lf", &st->midlon, &st->midlat);
+					}
+
+					if ((k = json_hash_get(j, "attribution")) != NULL) {
+						attribution = k->string;
+					}
+
+					if ((k = json_hash_get(j, "description")) != NULL) {
+						description = k->string;
+					}
+
+					if ((k = json_hash_get(j, "name")) != NULL) {
+						const std::string name_tmp = k->string;
+						if (name.size() == 0) {
+							name = name_tmp;
+						} else {
+							name += " + " + name_tmp;
+						}
+					}
+
+					if ((k = json_hash_get(j, "bounds")) != NULL) {
+						const std::string bounds = k->string;
+						const unsigned char *s = (const unsigned char *) bounds.c_str();
+						if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) {
+							st->minlon = min(minlon, st->minlon);
+							st->maxlon = max(maxlon, st->maxlon);
+							st->minlat = min(minlat, st->minlat);
+							st->maxlat = max(maxlat, st->maxlat);
+						}
+					}
+				}
+			}
+			json_free(j);
+			json_end(jp);
+			fclose(f);
+		}
+
 		delete r;
 	}
 }
 
 void usage(char **argv) {
-	fprintf(stderr, "Usage: %s [-f] [-i] [-pk] [-c joins.csv] [-x exclude ...] -o new.mbtiles source.mbtiles ...\n", argv[0]);
+	fprintf(stderr, "Usage: %s [-f] [-i] [-pk] [-pC] [-c joins.csv] [-x exclude ...] -o new.mbtiles source.mbtiles ...\n", argv[0]);
 	exit(EXIT_FAILURE);
 }
 
@@ -705,7 +985,9 @@ void readcsv(char *fn, std::vector<std::string> &header, std::map<std::string, s
 }
 
 int main(int argc, char **argv) {
-	char *outfile = NULL;
+	char *out_mbtiles = NULL;
+	char *out_dir = NULL;
+	sqlite3 *outdb = NULL;
 	char *csv = NULL;
 	int force = 0;
 	int ifmatched = 0;
@@ -728,10 +1010,14 @@ int main(int argc, char **argv) {
 	extern char *optarg;
 	int i;
 
-	while ((i = getopt(argc, argv, "fo:c:x:ip:l:L:A:N:n:")) != -1) {
+	while ((i = getopt(argc, argv, "fo:e:c:x:ip:l:L:A:N:n:")) != -1) {
 		switch (i) {
 		case 'o':
-			outfile = optarg;
+			out_mbtiles = optarg;
+			break;
+
+		case 'e':
+			out_dir = optarg;
 			break;
 
 		case 'f':
@@ -757,6 +1043,8 @@ int main(int argc, char **argv) {
 		case 'p':
 			if (strcmp(optarg, "k") == 0) {
 				pk = true;
+			} else if (strcmp(optarg, "C") == 0) {
+				pC = true;
 			} else {
 				fprintf(stderr, "%s: Unknown option for -p%s\n", argv[0], optarg);
 				exit(EXIT_FAILURE);
@@ -790,15 +1078,27 @@ int main(int argc, char **argv) {
 		}
 	}
 
-	if (argc - optind < 1 || outfile == NULL) {
+	if (argc - optind < 1) {
 		usage(argv);
 	}
 
+	if (out_mbtiles == NULL && out_dir == NULL) {
+		fprintf(stderr, "%s: must specify -o out.mbtiles or -e directory\n", argv[0]);
+		usage(argv);
+	}
+
+	if (out_mbtiles != NULL && out_dir != NULL) {
+		fprintf(stderr, "%s: Options -o and -e cannot be used together\n", argv[0]);
+		usage(argv);
+	}
+
+	if (out_mbtiles != NULL) {
 		if (force) {
-		unlink(outfile);
+			unlink(out_mbtiles);
+		}
+		outdb = mbtiles_open(out_mbtiles, argv, 0);
 	}
 
-	sqlite3 *outdb = mbtiles_open(outfile, argv, 0);
 	struct stats st;
 	memset(&st, 0, sizeof(st));
 	st.minzoom = st.minlat = st.minlon = INT_MAX;
@@ -825,7 +1125,7 @@ int main(int argc, char **argv) {
 		*rr = r;
 	}
 
-	decode(readers, csv, layermap, outdb, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name);
+	decode(readers, csv, layermap, outdb, out_dir, &st, header, mapping, exclude, ifmatched, attribution, description, keep_layers, remove_layers, name);
 
 	if (set_attribution.size() != 0) {
 		attribution = set_attribution;
@@ -837,8 +1137,11 @@ int main(int argc, char **argv) {
 		name = set_name;
 	}
 
-	mbtiles_write_metadata(outdb, NULL, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str());
+	mbtiles_write_metadata(outdb, out_dir, name.c_str(), st.minzoom, st.maxzoom, st.minlat, st.minlon, st.maxlat, st.maxlon, st.midlat, st.midlon, 0, attribution.size() != 0 ? attribution.c_str() : NULL, layermap, true, description.c_str());
+
+	if (outdb != NULL) {
 		mbtiles_close(outdb, argv);
+	}
 
 	return 0;
 }

tile.cpp

@@ -22,7 +22,7 @@
 #include <time.h>
 #include "mvt.hpp"
 #include "mbtiles.hpp"
-#include "rawtiles.hpp"
+#include "dirtiles.hpp"
 #include "geometry.hpp"
 #include "tile.hpp"
 #include "pool.hpp"
@@ -1921,7 +1921,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 					if (outdb != NULL) {
 						mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
 					} else if (outdir != NULL) {
-						write_raw_tile(outdir, z, tx, ty, compressed);
+						dir_write_tile(outdir, z, tx, ty, compressed);
 					}
 
 					if (pthread_mutex_unlock(&db_lock) != 0) {