Limit the size of the string pools and trees to fit in memory

Makefile

@@ -56,7 +56,7 @@ tippecanoe-enumerate: enumerate.o
 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
-tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o
+tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o

main.cpp

@@ -105,6 +105,7 @@ struct source {
 size_t CPUS;
 size_t TEMP_FILES;
 long long MAX_FILES;
+size_t memsize;
 static long long diskfree;
 char **av;
 
@@ -1031,17 +1032,8 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d
 	}
 }
 
-void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic<long long> *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
+static size_t calc_memsize() {
-	// Run through the index and geometry for each reader,
+	size_t mem;
-	// splitting the contents out by index into as many
-	// sub-files as we can write to simultaneously.
-
-	// Then sort each of those by index, recursively if it is
-	// too big to fit in memory.
-
-	// Then concatenate each of the sub-outputs into a final output.
-
-	long long mem;
 
 #ifdef __APPLE__
 	int64_t hw_memsize;
@@ -1062,6 +1054,21 @@ void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FI
 	mem = (long long) pages * pagesize;
 #endif
 
+	return mem;
+}
+
+void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic<long long> *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
+	// Run through the index and geometry for each reader,
+	// splitting the contents out by index into as many
+	// sub-files as we can write to simultaneously.
+
+	// Then sort each of those by index, recursively if it is
+	// too big to fit in memory.
+
+	// Then concatenate each of the sub-outputs into a final output.
+
+	long long mem = memsize;
+
 	// Just for code coverage testing. Deeply recursive sorting is very slow
 	// compared to sorting in memory.
 	if (additional[A_PREFER_RADIX_SORT]) {
@@ -2650,6 +2657,8 @@ int main(int argc, char **argv) {
 	int files_open_at_start;
 	json_object *filter = NULL;
 
+	memsize = calc_memsize();
+
 	for (i = 0; i < 256; i++) {
 		prevent[i] = 0;
 		additional[i] = 0;

main.hpp

@@ -47,6 +47,7 @@ extern int extra_detail;
 
 extern size_t CPUS;
 extern size_t TEMP_FILES;
+extern size_t memsize;
 
 extern size_t max_tile_size;
 extern size_t max_tile_features;

pool.cpp

@@ -3,6 +3,7 @@
 #include <string.h>
 #include <limits.h>
 #include <math.h>
+#include "main.hpp"
 #include "memfile.hpp"
 #include "pool.hpp"
 #include "errors.hpp"
@@ -74,6 +75,23 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
 		}
 	}
 
+	if ((size_t) (poolfile->off + treefile->off) > memsize / CPUS / 2) {
+		// If the pool and search tree get to be larger than physical memory,
+		// then searching will start thrashing. Just append the new string
+		// to the pool rather than letting the tree grow any further.
+
+		long long off = poolfile->off;
+		if (memfile_write(poolfile, &type, 1) < 0) {
+			perror("memfile write");
+			exit(EXIT_WRITE);
+		}
+		if (memfile_write(poolfile, (void *) s, strlen(s) + 1) < 0) {
+			perror("memfile write");
+			exit(EXIT_WRITE);
+		}
+		return off;
+	}
+
 	// *sp is probably in the memory-mapped file, and will move if the file grows.
 	long long ssp;
 	if (sp == &treefile->tree) {

tile-join.cpp

@@ -25,7 +25,6 @@
 #include <pthread.h>
 #include "mvt.hpp"
 #include "projection.hpp"
-#include "pool.hpp"
 #include "mbtiles.hpp"
 #include "geometry.hpp"
 #include "dirtiles.hpp"