When the pool gets too big, switch to just the file, not memory

main.cpp

@@ -116,7 +116,7 @@ void checkdisk(std::vector<struct reader> *r) {
 	for (size_t i = 0; i < r->size(); i++) {
 		// Pool and tree are used once.
 		// Geometry and index will be duplicated during sorting and tiling.
-		used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->map.size() + (*r)[i].treefile->map.size();
+		used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->off + (*r)[i].treefile->off;
 	}
 
 	static int warned = 0;
@@ -1840,15 +1840,48 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 	std::atomic<long long> poolpos(0);
 
 	for (size_t i = 0; i < CPUS; i++) {
-		if (readers[i].poolfile->map.size() > 0) {
+		// If the memfile is not done yet, it is in memory, so just copy the memory.
-			if (fwrite(readers[i].poolfile->map.c_str(), readers[i].poolfile->map.size(), 1, poolfile) != 1) {
+		// Otherwise, we need to merge memory and file.
-				perror("Reunify string pool");
+
+		if (readers[i].poolfile->fp == NULL) {
+			// still in memory
+
+			if (readers[i].poolfile->map.size() > 0) {
+				if (fwrite(readers[i].poolfile->map.c_str(), readers[i].poolfile->map.size(), 1, poolfile) != 1) {
+					perror("Reunify string pool");
+					exit(EXIT_WRITE);
+				}
+			}
+
+			pool_off[i] = poolpos;
+			poolpos += readers[i].poolfile->map.size();
+		} else {
+			// split into memory and file
+
+			if (fflush(readers[i].poolfile->fp) != 0) {
+				perror("fflush poolfile");
 				exit(EXIT_WRITE);
 			}
+
+			char *s = (char *) mmap(NULL, readers[i].poolfile->off, PROT_READ, MAP_PRIVATE, readers[i].poolfile->fd, 0);
+			if (s == MAP_FAILED) {
+				perror("mmap string pool for copy");
+				exit(EXIT_MEMORY);
+			}
+			madvise(s, readers[i].poolfile->off, MADV_SEQUENTIAL);
+			if (fwrite(s, sizeof(char), readers[i].poolfile->off, poolfile) != readers[i].poolfile->off) {
+				perror("Reunify string pool (split)");
+				exit(EXIT_WRITE);
+			}
+			if (munmap(s, readers[i].poolfile->off) != 0) {
+				perror("unmap string pool for copy");
+				exit(EXIT_MEMORY);
+			}
+
+			pool_off[i] = poolpos;
+			poolpos += readers[i].poolfile->off;
 		}
 
-		pool_off[i] = poolpos;
-		poolpos += readers[i].poolfile->map.size();
 		memfile_close(readers[i].poolfile);
 	}
 

memfile.cpp

@@ -19,17 +19,27 @@ struct memfile *memfile_open(int fd) {
 
 	mf->fd = fd;
 	mf->tree = 0;
+	mf->off = 0;
 
 	return mf;
 }
 
 int memfile_close(struct memfile *file) {
-	if (write(file->fd, file->map.c_str(), file->map.size()) != (ssize_t) file->map.size()) {
+	// If it isn't full yet, flush out the string to the file now.
-		return -1;
+	// If it is full, close out the buffered file writer.
-	}
 
-	if (file->fd >= 0) {
+	if (file->fp == NULL) {
-		if (close(file->fd) != 0) {
+		if (write(file->fd, file->map.c_str(), file->map.size()) != (ssize_t) file->map.size()) {
+			return -1;
+		}
+
+		if (file->fd >= 0) {
+			if (close(file->fd) != 0) {
+				return -1;
+			}
+		}
+	} else {
+		if (fclose(file->fp) != 0) {
 			return -1;
 		}
 	}
@@ -39,6 +49,42 @@ int memfile_close(struct memfile *file) {
 }
 
 int memfile_write(struct memfile *file, void *s, long long len) {
-	file->map.append(std::string((char *) s, len));
+	// If it is full, append to the file.
+	// If it is not full yet, append to the string in memory.
+
+	if (file->fp != NULL) {
+		if (fwrite(s, sizeof(char), len, file->fp) != (size_t) len) {
+			return 0;
+		}
+		file->off += len;
+	} else {
+		file->map.append(std::string((char *) s, len));
+		file->off += len;
+	}
+
 	return len;
 }
+
+void memfile_full(struct memfile *file) {
+	// The file is full. Write out a copy of whatever has accumulated in memory
+	// to the file, and switch to appending to the file. Existing references
+	// into the memory still work.
+
+	if (file->fp != NULL) {
+		fprintf(stderr, "memfile marked full twice\n");
+		exit(EXIT_IMPOSSIBLE);
+	}
+
+	fprintf(stderr, "Tippecanoe string pool is full (%zu); switching to file\n", file->off);
+
+	file->fp = fdopen(file->fd, "wb");
+	if (file->fp == NULL) {
+		fprintf(stderr, "fdopen memfile: %s\n", strerror(errno));
+		exit(EXIT_OPEN);
+	}
+
+	if (fwrite(file->map.c_str(), sizeof(char), file->map.size(), file->fp) != file->map.size()) {
+		fprintf(stderr, "memfile write: %s\n", strerror(errno));
+		exit(EXIT_WRITE);
+	}
+}

memfile.hpp

@@ -3,15 +3,19 @@
 
 #include <atomic>
 #include <string>
+#include "errors.hpp"
 
 struct memfile {
 	int fd = 0;
 	std::string map;
 	unsigned long tree = 0;
+	FILE *fp = NULL;
+	size_t off = 0;
 };
 
 struct memfile *memfile_open(int fd);
 int memfile_close(struct memfile *file);
 int memfile_write(struct memfile *file, void *s, long long len);
+void memfile_full(struct memfile *file);
 
 #endif

pool.cpp

@@ -37,7 +37,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
 	size_t depth = 0;
 
 	// In typical data, traversal depth generally stays under 2.5x
-	size_t max = 3 * log(treefile->map.size() / sizeof(struct stringpool)) / log(2);
+	size_t max = 3 * log(treefile->off / sizeof(struct stringpool)) / log(2);
 	if (max < 30) {
 		max = 30;
 	}
@@ -61,8 +61,10 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
 		if (depth > max) {
 			// Search is very deep, so string is probably unique.
 			// Add it to the pool without adding it to the search tree.
+			// This might go either to memory or the file, depending on whether
+			// the pool is full yet.
 
-			long long off = poolfile->map.size();
+			long long off = poolfile->off;
 			if (memfile_write(poolfile, &type, 1) < 0) {
 				perror("memfile write");
 				exit(EXIT_WRITE);
@@ -75,12 +77,22 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
 		}
 	}
 
-	if ((size_t) (poolfile->map.size() + treefile->map.size()) > memsize / CPUS / 2) {
+	// Size of memory divided by 5 from observation of OOM errors on ECS
+	if ((size_t) (poolfile->off + treefile->off) > memsize / CPUS / 5) {
 		// If the pool and search tree get to be larger than physical memory,
-		// then searching will start thrashing. Just append the new string
+		// then searching will start thrashing. Switch to appending strings
-		// to the pool rather than letting the tree grow any further.
+		// to the file instead of keeping them in memory.
 
-		long long off = poolfile->map.size();
+		if (poolfile->fp == NULL) {
+			memfile_full(poolfile);
+		}
+	}
+
+	if (poolfile->fp != NULL) {
+		// We are now appending to the file, so don't try to keep tree references
+		// to the newly-added strings.
+
+		long long off = poolfile->off;
 		if (memfile_write(poolfile, &type, 1) < 0) {
 			perror("memfile write");
 			exit(EXIT_WRITE);
@@ -100,7 +112,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
 		ssp = ((char *) sp) - treefile->map.c_str();
 	}
 
-	long long off = poolfile->map.size();
+	long long off = poolfile->off;
 	if (memfile_write(poolfile, &type, 1) < 0) {
 		perror("memfile write");
 		exit(EXIT_WRITE);
@@ -110,7 +122,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
 		exit(EXIT_WRITE);
 	}
 
-	if (off >= LONG_MAX || treefile->map.size() >= LONG_MAX) {
+	if (off >= LONG_MAX || treefile->off >= LONG_MAX) {
 		// Tree or pool is bigger than 2GB
 		static bool warned = false;
 		if (!warned) {
@@ -125,7 +137,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
 	tsp.right = 0;
 	tsp.off = off;
 
-	long long p = treefile->map.size();
+	long long p = treefile->off;
 	if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) {
 		perror("memfile write");
 		exit(EXIT_WRITE);