Essentials of multithreaded tile-joining

tile-join.cpp

@@ -10,6 +10,7 @@
 #include <set>
 #include <zlib.h>
 #include <math.h>
+#include <pthread.h>
 #include "mvt.hpp"
 #include "projection.hpp"
 #include "pool.hpp"
@@ -28,7 +29,7 @@ struct stats {
 	double minlat, minlon, maxlat, maxlon;
 };
 
-void handle(std::string message, int z, unsigned x, unsigned y, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, std::vector<std::string> &header, std::map<std::string, std::vector<std::string> > &mapping, std::set<std::string> &exclude, int ifmatched, mvt_tile &outtile) {
+void handle(std::string message, int z, unsigned x, unsigned y, std::map<std::string, layermap_entry> &layermap, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, mvt_tile &outtile) {
 	mvt_tile tile;
 	int features_added = 0;
 
@@ -299,23 +300,62 @@ struct reader *begin_reading(char *fname) {
 	return r;
 }
 
-void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string> > &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution) {
+struct zxy {
-	std::vector<std::map<std::string, layermap_entry> > layermaps;
+	long long z;
-	for (size_t i = 0; i < CPUS; i++) {
+	long long x;
-		layermaps.push_back(std::map<std::string, layermap_entry>());
+	long long y;
+
+	zxy(long long _z, long long _x, long long _y) {
+		z = _z;
+		x = _x;
+		y = _y;
 	}
 
+	bool operator<(zxy const &other) const {
+		if (z < other.z) {
+			return true;
+		}
+		if (z > other.z) {
+			return false;
+		}
+
+		if (x < other.x) {
+			return true;
+		}
+		if (x > other.x) {
+			return false;
+		}
+
+		if (y < other.y) {
+			return true;
+		}
+
+		return false;
+	}
+};
+
+struct arg {
+	std::map<zxy, std::vector<std::string>> inputs;
+	std::map<zxy, std::string> outputs;
+
+	std::map<std::string, layermap_entry> *layermap;
+
+	std::vector<std::string> *header;
+	std::map<std::string, std::vector<std::string>> *mapping;
+	std::set<std::string> *exclude;
+	int ifmatched;
+};
+
+void *join_worker(void *v) {
+	arg *a = (arg *) v;
+
+	for (auto ai = a->inputs.begin(); ai != a->inputs.end(); ++ai) {
 		mvt_tile tile;
 
-	while (readers != NULL && readers->zoom < 32) {
+		for (size_t i = 0; i < ai->second.size(); i++) {
-		reader *r = readers;
+			handle(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), *(a->exclude), a->ifmatched, tile);
-		readers = readers->next;
+		}
-		r->next = NULL;
 
-		fprintf(stderr, "%lld/%lld/%lld   \r", r->zoom, r->x, r->y);
-		handle(r->data, r->zoom, r->x, r->y, layermaps[0], outdb, header, mapping, exclude, ifmatched, tile);
-
-		if (readers == NULL || readers->zoom != r->zoom || readers->x != r->x || readers->y != r->y) {
 		bool anything = false;
 		for (size_t i = 0; i < tile.layers.size(); i++) {
 			if (tile.layers[i].features.size() > 0) {
@@ -328,13 +368,86 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 			std::string compressed = tile.encode();
 
 			if (!pk && compressed.size() > 500000) {
-					fprintf(stderr, "Tile %lld/%lld/%lld size is %lld, >500000. Skipping this tile\n.", r->zoom, r->x, r->y, (long long) compressed.size());
+				fprintf(stderr, "Tile %lld/%lld/%lld size is %lld, >500000. Skipping this tile\n.", ai->first.z, ai->first.x, ai->first.y, (long long) compressed.size());
 			} else {
-					mbtiles_write_tile(outdb, r->zoom, r->x, r->y, compressed.data(), compressed.size());
+				a->outputs.insert(std::pair<zxy, std::string>(ai->first, compressed));
+			}
 		}
 	}
 
-			tile = mvt_tile();
+	return NULL;
+}
+
+void handle_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched) {
+	pthread_t pthreads[CPUS];
+	std::vector<arg> args;
+
+	for (size_t i = 0; i < CPUS; i++) {
+		args.push_back(arg());
+
+		args[i].layermap = &layermaps[i];
+		args[i].header = &header;
+		args[i].mapping = &mapping;
+		args[i].exclude = &exclude;
+		args[i].ifmatched = ifmatched;
+	}
+
+	size_t count = 0;
+	// XXX Be more careful to distribute tasks evenly across CPUs
+	for (auto ai = tasks.begin(); ai != tasks.end(); ++ai) {
+		args[count].inputs.insert(*ai);
+		count = (count + 1) % CPUS;
+
+		if (ai == tasks.begin()) {
+			fprintf(stderr, "%lld/%lld/%lld  \r", ai->first.z, ai->first.x, ai->first.y);
+		}
+	}
+
+	for (size_t i = 0; i < CPUS; i++) {
+		if (pthread_create(&pthreads[i], NULL, join_worker, &args[i]) != 0) {
+			perror("pthread_create");
+			exit(EXIT_FAILURE);
+		}
+	}
+
+	for (int i = 0; i < CPUS; i++) {
+		void *retval;
+
+		if (pthread_join(pthreads[i], &retval) != 0) {
+			perror("pthread_join");
+		}
+
+		for (auto ai = args[i].outputs.begin(); ai != args[i].outputs.end(); ++ai) {
+			mbtiles_write_tile(outdb, ai->first.z, ai->first.x, ai->first.y, ai->second.data(), ai->second.size());
+		}
+	}
+}
+
+void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution) {
+	std::vector<std::map<std::string, layermap_entry>> layermaps;
+	for (size_t i = 0; i < CPUS; i++) {
+		layermaps.push_back(std::map<std::string, layermap_entry>());
+	}
+
+	std::map<zxy, std::vector<std::string>> tasks;
+
+	while (readers != NULL && readers->zoom < 32) {
+		reader *r = readers;
+		readers = readers->next;
+		r->next = NULL;
+
+		zxy tile = zxy(r->zoom, r->x, r->y);
+		if (tasks.count(tile) == 0) {
+			tasks.insert(std::pair<zxy, std::vector<std::string>>(tile, std::vector<std::string>()));
+		}
+		auto f = tasks.find(tile);
+		f->second.push_back(r->data);
+
+		if (readers == NULL || readers->zoom != r->zoom || readers->x != r->x || readers->y != r->y) {
+			if (tasks.size() > 100 * CPUS) {
+				handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched);
+				tasks.clear();
+			}
 		}
 
 		if (sqlite3_step(r->stmt) == SQLITE_ROW) {
@@ -363,6 +476,7 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 		*rr = r;
 	}
 
+	handle_tasks(tasks, layermaps, outdb, header, mapping, exclude, ifmatched);
 	layermap = merge_layermaps(layermaps);
 
 	struct reader *next;