Reduce tile-join overzooming memory usage (#162)

* 16 bits is enough for tile numbers

* Revert "16 bits is enough for tile numbers"

This reverts commit 71a0c4e1cf.

* Check what child tiles each overzoomed tile will have

and don't queue further overzooming of empty tiles

* Clean up naming; use std::move to avoid copying large arrays

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.37.0
+
+* Speed up tile-join overzooming and make it use less memory, by not including empty child tiles in the enumeration
+
 # 2.36.0
 
 * Make tile-join distrust the source tilesets' metadata maxzoom and minzoom

clip.cpp

@@ -753,7 +753,8 @@ static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<d
 }
 
 std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
-		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress) {
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
 	mvt_tile tile;
 
 	try {
@@ -767,11 +768,12 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
 		exit(EXIT_PROTOBUF);
 	}
 
-	return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress);
+	return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles);
 }
 
 std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
-		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress) {
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
 	mvt_tile outtile;
 
 	for (auto const &layer : tile.layers) {
@@ -890,6 +892,28 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
 		}
 	}
 
+	if (next_overzoomed_tiles != NULL) {
+		// will any child tiles have features in them?
+		// find out recursively from the tile we just made.
+		//
+		// (yes, we should keep them instead of remaking them
+		// later, but that first requires figuring out where to
+		// keep them.)
+
+		if (outtile.layers.size() > 0) {
+			for (size_t x = 0; x < 2; x++) {
+				for (size_t y = 0; y < 2; y++) {
+					std::string child = overzoom(outtile, nz, nx, ny,
+								     nz + 1, nx * 2 + x, ny * 2 + y,
+								     detail, buffer, keep, false, NULL);
+					if (child.size() > 0) {
+						next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y);
+					}
+				}
+			}
+		}
+	}
+
 	if (outtile.layers.size() > 0) {
 		std::string pbf = outtile.encode();
 

geometry.hpp

@@ -99,9 +99,11 @@ int pnpoly(const drawvec &vert, size_t start, size_t nvert, long long testx, lon
 double distance_from_line(long long point_x, long long point_y, long long segA_x, long long segA_y, long long segB_x, long long segB_y);
 
 std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
-		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress);
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);
 
 std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
-		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress);
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);
 
 #endif

overzoom.cpp

@@ -91,7 +91,7 @@ int main(int argc, char **argv) {
 		exit(EXIT_FAILURE);
 	}
 
-	std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, true);
+	std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, true, NULL);
 	fwrite(out.c_str(), sizeof(char), out.size(), f);
 	fclose(f);
 

tile-join.cpp

@@ -434,7 +434,8 @@ struct tileset_reader {
 	// for overzooming
 	int maxzoom_so_far = -1;
 	std::vector<std::pair<unsigned, unsigned>> tiles_at_maxzoom_so_far;
-	std::vector<std::pair<unsigned, unsigned>> overzoomed_tiles;
+	std::vector<std::pair<unsigned, unsigned>> overzoomed_tiles;	   // tiles at `zoom`
+	std::vector<std::pair<unsigned, unsigned>> next_overzoomed_tiles;  // tiles at `zoom + 1`
 	bool overzoom_consumed_at_this_zoom = false;
 
 	// parent tile cache
@@ -642,12 +643,24 @@ struct tileset_reader {
 
 		long long scale = (1LL << zoom) / (1LL << maxzoom_so_far);
 
-		for (auto const &xy : tiles_at_maxzoom_so_far) {
+		// If this is the first overzoomed level, we don't know yet
-			for (long long xx = 0; xx < scale; xx++) {
+		// which tiles will be useful, so spell out all 4 child tiles
-				for (long long yy = 0; yy < scale; yy++) {
+		// from each parent tile.
-					overzoomed_tiles.push_back(std::pair<unsigned, unsigned>(xy.first * scale + xx, xy.second * scale + yy));
+		//
+		// If it is further overzoomed than that, we have a list of
+		// which child tiles will have features in them, so use that.
+
+		if (zoom == maxzoom_so_far + 1) {
+			for (auto const &xy : tiles_at_maxzoom_so_far) {
+				for (long long xx = 0; xx < scale; xx++) {
+					for (long long yy = 0; yy < scale; yy++) {
+						overzoomed_tiles.push_back(std::pair<unsigned, unsigned>(xy.first * scale + xx, xy.second * scale + yy));
+					}
 				}
 			}
+		} else {
+			overzoomed_tiles = std::move(next_overzoomed_tiles);
+			next_overzoomed_tiles.clear();
 		}
 
 		std::sort(overzoomed_tiles.begin(), overzoomed_tiles.end(), tilecmp);
@@ -769,7 +782,7 @@ struct tileset_reader {
 		}
 
 		if (source.layers.size() != 0) {
-			std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false);
+			std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles);
 			return ret;
 		}
 

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.36.0"
+#define VERSION "v2.37.0"
 
 #endif