diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 16:35:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Limit the size of the string pools and trees to fit in memory

Browse Source
This commit is contained in:
Erica Fischer
2023-01-03 11:10:52 -08:00
parent f59f8588eb
commit 69bf5b324b
5 changed files with 40 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pool.cpp
+18
View File
@@ -3,6 +3,7 @@
#include <string.h>
#include <limits.h>
#include <math.h>
#include "main.hpp"
#include "memfile.hpp"
#include "pool.hpp"
#include "errors.hpp"
@@ -74,6 +75,23 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
}
}
if ((size_t) (poolfile->off + treefile->off) > memsize / CPUS / 2) {
// If the pool and search tree get to be larger than physical memory,
// then searching will start thrashing. Just append the new string
// to the pool rather than letting the tree grow any further.
long long off = poolfile->off;
if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write");
exit(EXIT_WRITE);
}
if (memfile_write(poolfile, (void *) s, strlen(s) + 1) < 0) {
perror("memfile write");
exit(EXIT_WRITE);
}
return off;
}
// *sp is probably in the memory-mapped file, and will move if the file grows.
long long ssp;
if (sp == &treefile->tree) {