Speed up overzooming in tile-join (#147)

* Clip away entire features by bbox. Avoid unnecessary recompression.

* Move parent tile decoding in tile-join out of overzoom proper

* An ever-growing cache of parent tiles

* Limit the size of the cache

* Remove the current reader *before* checking if we can run the queue

* Clean up

* Add missing #include

* Add comment

* When the tile-join cache fills up, evict the least recently used

* Fix microsecond math

* Factoring out tile-join's cache for testing

* Add unit tests for tile-join cache

* Update changelog and version

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.34.0
+
+* Improve speed of overzooming in tile-join
+
 # 2.33.0
 
 * Further reduce memory usage of --no-simplification-of-shared-nodes by calculating the list of shared nodes globally using temporary files rather than in memory for each individual tile

Makefile

@@ -73,7 +73,7 @@ tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpul
 tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
-unit: unit.o text.o sort.o
+unit: unit.o text.o sort.o mvt.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 tippecanoe-overzoom: overzoom.o mvt.o clip.o

clip.cpp

@@ -1,6 +1,8 @@
+#include <stdlib.h>
 #include <mapbox/geometry/point.hpp>
 #include <mapbox/geometry/multi_polygon.hpp>
 #include <mapbox/geometry/wagyu/wagyu.hpp>
+#include <limits.h>
 #include "geometry.hpp"
 #include "errors.hpp"
 #include "compression.hpp"
@@ -751,11 +753,11 @@ static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<d
 }
 
 std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
-		     int detail, int buffer, std::set<std::string> const &keep) {
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress) {
-	mvt_tile tile, outtile;
+	mvt_tile tile;
-	bool was_compressed;
 
 	try {
+		bool was_compressed;
 		if (!tile.decode(s, was_compressed)) {
 			fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", oz, ox, oy);
 			exit(EXIT_MVT);
@@ -765,6 +767,13 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
 		exit(EXIT_PROTOBUF);
 	}
 
+	return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress);
+}
+
+std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress) {
+	mvt_tile outtile;
+
 	for (auto const &layer : tile.layers) {
 		mvt_layer outlayer = mvt_layer();
 
@@ -813,6 +822,23 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
 
 			// Clip to output tile
 
+			long long xmin = LLONG_MAX;
+			long long ymin = LLONG_MAX;
+			long long xmax = LLONG_MIN;
+			long long ymax = LLONG_MIN;
+
+			for (auto const &g : geom) {
+				xmin = std::min(xmin, g.x);
+				ymin = std::min(ymin, g.y);
+				xmax = std::max(xmax, g.x);
+				ymax = std::max(ymax, g.y);
+			}
+
+			long long b = outtilesize * buffer / 256;
+			if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) {
+				continue;
+			}
+
 			if (t == VT_LINE) {
 				geom = clip_lines(geom, nz, buffer);
 			} else if (t == VT_POLYGON) {
@@ -866,8 +892,13 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
 
 	if (outtile.layers.size() > 0) {
 		std::string pbf = outtile.encode();
+
 		std::string compressed;
+		if (do_compress) {
 			compress(pbf, compressed, true);
+		} else {
+			compressed = pbf;
+		}
 
 		return compressed;
 	} else {

dirtiles.hpp

@@ -1,6 +1,7 @@
 #include <string>
 #include <vector>
 #include <sys/stat.h>
+#include "mbtiles.hpp"
 
 #ifndef DIRTILES_HPP
 #define DIRTILES_HPP

geometry.hpp

@@ -7,6 +7,7 @@
 #include <string>
 #include <sqlite3.h>
 #include <stdio.h>
+#include <mvt.hpp>
 
 #define VT_POINT 1
 #define VT_LINE 2
@@ -97,7 +98,10 @@ void visvalingam(drawvec &ls, size_t start, size_t end, double threshold, size_t
 int pnpoly(const drawvec &vert, size_t start, size_t nvert, long long testx, long long testy);
 double distance_from_line(long long point_x, long long point_y, long long segA_x, long long segA_y, long long segB_x, long long segB_y);
 
+std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress);
+
 std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
-		     int detail, int buffer, std::set<std::string> const &keep);
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress);
 
 #endif

overzoom.cpp

@@ -91,7 +91,7 @@ int main(int argc, char **argv) {
 		exit(EXIT_FAILURE);
 	}
 
-	std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep);
+	std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, true);
 	fwrite(out.c_str(), sizeof(char), out.size(), f);
 	fclose(f);
 

tile-cache.hpp

@@ -0,0 +1,46 @@
+#include <functional>
+#include "mvt.hpp"
+#include "dirtiles.hpp"	 // for zxy
+
+struct mvt_tile_seq {
+	mvt_tile tile;
+	size_t seq;
+};
+
+struct tile_cache {
+	std::map<zxy, mvt_tile_seq> overzoom_cache;
+	std::atomic<size_t> seq;
+	size_t capacity = 1000;
+
+	mvt_tile get(zxy parent_tile, std::function<mvt_tile(zxy)> getter) {
+		mvt_tile source;
+		auto f = overzoom_cache.find(parent_tile);
+		if (f == overzoom_cache.end()) {
+			if (overzoom_cache.size() >= capacity) {
+				// evict the oldest tile to make room
+
+				auto to_erase = overzoom_cache.begin();
+				for (auto here = overzoom_cache.begin(); here != overzoom_cache.end(); ++here) {
+					if (here->second.seq < to_erase->second.seq) {
+						to_erase = here;
+					}
+				}
+
+				overzoom_cache.erase(to_erase);
+			}
+
+			source = getter(parent_tile);
+
+			mvt_tile_seq to_cache;
+			to_cache.tile = source;
+			to_cache.seq = seq++;
+
+			overzoom_cache.emplace(parent_tile, to_cache);
+		} else {
+			f->second.seq = seq++;
+			source = f->second.tile;
+		}
+
+		return source;
+	}
+};

tile-join.cpp

@@ -33,6 +33,7 @@
 #include "csv.hpp"
 #include "text.hpp"
 #include "tile.hpp"
+#include "tile-cache.hpp"
 #include <fstream>
 #include <sstream>
 #include <algorithm>
@@ -435,6 +436,9 @@ struct tileset_reader {
 	std::vector<std::pair<unsigned, unsigned>> overzoomed_tiles;
 	bool overzoom_consumed_at_this_zoom = false;
 
+	// parent tile cache
+	tile_cache cache;
+
 	// for iterating mbtiles
 	sqlite3 *db = NULL;
 	sqlite3_stmt *stmt = NULL;
@@ -649,7 +653,7 @@ struct tileset_reader {
 		overzoom_consumed_at_this_zoom = false;
 	}
 
-	std::string get_tile(zxy tile) {
+	mvt_tile get_tile(zxy tile) {
 		std::string source;
 
 		if (db != NULL) {
@@ -683,7 +687,23 @@ struct tileset_reader {
 			source = dir_read_tile(dirbase, tile);
 		}
 
-		return source;
+		mvt_tile content;
+		if (source.size() == 0) {
+			return content;
+		}
+
+		try {
+			bool was_compressed;
+			if (!content.decode(source, was_compressed)) {
+				fprintf(stderr, "Couldn't parse tile %lld/%lld/%lld\n", tile.z, tile.x, tile.y);
+				exit(EXIT_MVT);
+			}
+		} catch (std::exception const &e) {
+			fprintf(stderr, "PBF decoding error in tile %lld/%lld/%lld\n", tile.z, tile.x, tile.y);
+			exit(EXIT_PROTOBUF);
+		}
+
+		return content;
 	}
 
 	// Sort in z/x/tms_y order, because that is the order of the
@@ -737,14 +757,18 @@ struct tileset_reader {
 			perror("pthread_mutex_lock");
 		}
 
-		std::string source = get_tile(parent_tile);
+		std::function<mvt_tile(zxy)> getter = [&](zxy tile) {
+			return get_tile(tile);
+		};
+
+		mvt_tile source = cache.get(parent_tile, getter);
 
 		if (pthread_mutex_unlock(&retrieve_lock) != 0) {
 			perror("pthread_mutex_unlock");
 		}
 
-		if (source.size() != 0) {
+		if (source.layers.size() != 0) {
-			std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>());
+			std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false);
 			return ret;
 		}
 
@@ -1021,13 +1045,7 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
 			f->second.push_back(current.second);
 		}
 
-		if (readers == NULL || readers->zoom != current.first.z || readers->x != current.first.x || readers->y != current.first.y) {
+		// Advance the tileset_reader that we just added as a task.
-			if (tasks.size() > 100 * CPUS) {
-				dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers);
-				tasks.clear();
-			}
-		}
-
 		// The reason this prefetches is so the tileset_reader queue can be
 		// priority-ordered, so the one with the next relevant tile
 		// is first in line.
@@ -1039,6 +1057,16 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
 		readers = readers->next;
 		r->next = NULL;
 
+		// Is the next tileset_reader on the tileset_reader queue looking at a different tile?
+		// Then this tile is done and we can safely run the output queue.
+
+		if (readers == NULL || readers->zoom != current.first.z || readers->x != current.first.x || readers->y != current.first.y) {
+			if (tasks.size() > 10 * CPUS) {
+				dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers);
+				tasks.clear();
+			}
+		}
+
 		// put the tileset_reader back onto the queue,
 		// in whatever sequence its next tile calls for
 

unit.cpp

@@ -2,6 +2,7 @@
 #include "catch/catch.hpp"
 #include "text.hpp"
 #include "sort.hpp"
+#include "tile-cache.hpp"
 #include <unistd.h>
 #include <limits.h>
 
@@ -65,3 +66,36 @@ TEST_CASE("External quicksort", "fqsort") {
 	fclose(f);
 	REQUIRE(nread == written);
 }
+
+mvt_tile mock_get_tile(zxy tile) {
+	mvt_layer l;
+	l.name = std::to_string(tile.z) + "/" + std::to_string(tile.x) + "/" + std::to_string(tile.y);
+	mvt_tile t;
+	t.layers.push_back(l);
+	return t;
+}
+
+TEST_CASE("Tile-join cache", "tile cache") {
+	tile_cache tc;
+	tc.capacity = 5;
+
+	REQUIRE(tc.get(zxy(11, 327, 791), mock_get_tile).layers[0].name == "11/327/791");
+	REQUIRE(tc.get(zxy(11, 5, 7), mock_get_tile).layers[0].name == "11/5/7");
+	REQUIRE(tc.get(zxy(11, 5, 8), mock_get_tile).layers[0].name == "11/5/8");
+	REQUIRE(tc.get(zxy(11, 5, 9), mock_get_tile).layers[0].name == "11/5/9");
+	REQUIRE(tc.get(zxy(11, 5, 10), mock_get_tile).layers[0].name == "11/5/10");
+	REQUIRE(tc.get(zxy(11, 327, 791), mock_get_tile).layers[0].name == "11/327/791");
+	REQUIRE(tc.overzoom_cache.size() == 5);
+	REQUIRE(tc.overzoom_cache.find(zxy(11, 327, 791)) != tc.overzoom_cache.end());
+	REQUIRE(tc.overzoom_cache.find(zxy(11, 5, 7)) != tc.overzoom_cache.end());
+
+	// verify that additional gets evict the least-recently-used elements
+
+	REQUIRE(tc.get(zxy(11, 5, 11), mock_get_tile).layers[0].name == "11/5/11");
+	REQUIRE(tc.overzoom_cache.size() == 5);
+	REQUIRE(tc.overzoom_cache.find(zxy(11, 5, 7)) == tc.overzoom_cache.end());
+
+	REQUIRE(tc.get(zxy(11, 5, 12), mock_get_tile).layers[0].name == "11/5/12");
+	REQUIRE(tc.overzoom_cache.size() == 5);
+	REQUIRE(tc.overzoom_cache.find(zxy(11, 5, 8)) == tc.overzoom_cache.end());
+}

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.33.0"
+#define VERSION "v2.34.0"
 
 #endif