Getting ready for multithreaded tile-joining

main.cpp

@@ -856,37 +856,6 @@ void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE
 	}
 }
 
-std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps) {
-	std::map<std::string, layermap_entry> out;
-
-	for (size_t i = 0; i < maps.size(); i++) {
-		for (auto map = maps[i].begin(); map != maps[i].end(); ++map) {
-			if (out.count(map->first) == 0) {
-				out.insert(std::pair<std::string, layermap_entry>(map->first, layermap_entry(out.size())));
-			}
-
-			auto out_entry = out.find(map->first);
-			if (out_entry == out.end()) {
-				fprintf(stderr, "Internal error merging layers\n");
-				exit(EXIT_FAILURE);
-			}
-
-			for (auto fk = map->second.file_keys.begin(); fk != map->second.file_keys.end(); ++fk) {
-				out_entry->second.file_keys.insert(*fk);
-			}
-
-			if (additional[A_CALCULATE_FEATURE_DENSITY]) {
-				type_and_string tas;
-				tas.type = VT_NUMBER;
-				tas.string = "tippecanoe_feature_density";
-				out_entry->second.file_keys.insert(tas);
-			}
-		}
-	}
-
-	return out;
-}
-
 int read_input(std::vector<source> &sources, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution) {
 	int ret = EXIT_SUCCESS;
 
@@ -1728,6 +1697,14 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
 	}
 
 	std::map<std::string, layermap_entry> merged_lm = merge_layermaps(layermaps);
+	if (additional[A_CALCULATE_FEATURE_DENSITY]) {
+		for (auto ai = merged_lm.begin(); ai != merged_lm.end(); ++ai) {
+			type_and_string tas;
+			tas.type = VT_NUMBER;
+			tas.string = "tippecanoe_feature_density";
+			ai->second.file_keys.insert(tas);
+		}
+	}
 
 	for (auto ai = merged_lm.begin(); ai != merged_lm.end(); ++ai) {
 		ai->second.minzoom = minzoom;

mbtiles.cpp

@@ -295,3 +295,37 @@ void mbtiles_close(sqlite3 *outdb, char **argv) {
 		exit(EXIT_FAILURE);
 	}
 }
+
+std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps) {
+	std::map<std::string, layermap_entry> out;
+
+	for (size_t i = 0; i < maps.size(); i++) {
+		for (auto map = maps[i].begin(); map != maps[i].end(); ++map) {
+			if (out.count(map->first) == 0) {
+				out.insert(std::pair<std::string, layermap_entry>(map->first, layermap_entry(out.size())));
+				auto out_entry = out.find(map->first);
+				out_entry->second.minzoom = map->second.minzoom;
+				out_entry->second.maxzoom = map->second.maxzoom;
+			}
+
+			auto out_entry = out.find(map->first);
+			if (out_entry == out.end()) {
+				fprintf(stderr, "Internal error merging layers\n");
+				exit(EXIT_FAILURE);
+			}
+
+			for (auto fk = map->second.file_keys.begin(); fk != map->second.file_keys.end(); ++fk) {
+				out_entry->second.file_keys.insert(*fk);
+			}
+
+			if (map->second.minzoom < out_entry->second.minzoom) {
+				out_entry->second.minzoom = map->second.minzoom;
+			}
+			if (map->second.maxzoom > out_entry->second.maxzoom) {
+				out_entry->second.maxzoom = map->second.maxzoom;
+			}
+		}
+	}
+
+	return out;
+}

mbtiles.hpp

@@ -25,3 +25,5 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *fname, int minzoom, int
 void mbtiles_close(sqlite3 *outdb, char **argv);
 
 void aprintf(std::string *buf, const char *format, ...);
+
+std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps);

tile-join.cpp

@@ -19,6 +19,7 @@
 std::string dequote(std::string s);
 
 bool pk = false;
+int CPUS;
 
 struct stats {
 	int minzoom;
@@ -299,6 +300,11 @@ struct reader *begin_reading(char *fname) {
 }
 
 void decode(struct reader *readers, char *map, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string> > &mapping, std::set<std::string> &exclude, int ifmatched, std::string &attribution) {
+	std::vector<std::map<std::string, layermap_entry> > layermaps;
+	for (size_t i = 0; i < CPUS; i++) {
+		layermaps.push_back(std::map<std::string, layermap_entry>());
+	}
+
 	mvt_tile tile;
 
 	while (readers != NULL && readers->zoom < 32) {
@@ -307,7 +313,7 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 		r->next = NULL;
 
 		fprintf(stderr, "%lld/%lld/%lld   \r", r->zoom, r->x, r->y);
-		handle(r->data, r->zoom, r->x, r->y, layermap, outdb, header, mapping, exclude, ifmatched, tile);
+		handle(r->data, r->zoom, r->x, r->y, layermaps[0], outdb, header, mapping, exclude, ifmatched, tile);
 
 		if (readers == NULL || readers->zoom != r->zoom || readers->x != r->x || readers->y != r->y) {
 			bool anything = false;
@@ -357,6 +363,8 @@ void decode(struct reader *readers, char *map, std::map<std::string, layermap_en
 		*rr = r;
 	}
 
+	layermap = merge_layermaps(layermaps);
+
 	struct reader *next;
 	for (struct reader *r = readers; r != NULL; r = next) {
 		next = r->next;
@@ -496,6 +504,11 @@ int main(int argc, char **argv) {
 	int force = 0;
 	int ifmatched = 0;
 
+	CPUS = sysconf(_SC_NPROCESSORS_ONLN);
+	if (CPUS < 1) {
+		CPUS = 1;
+	}
+
 	std::vector<std::string> header;
 	std::map<std::string, std::vector<std::string> > mapping;