diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-03 00:45:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

When the pool gets too big, switch to just the file, not memory

Browse Source
This commit is contained in:
Erica Fischer
2023-01-17 11:04:26 -08:00
parent 2695db5ebf
commit 1ef0f260f1
4 changed files with 116 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+39 -6
View File
@@ -116,7 +116,7 @@ void checkdisk(std::vector<struct reader> *r) {
for (size_t i = 0; i < r->size(); i++) {
// Pool and tree are used once.
// Geometry and index will be duplicated during sorting and tiling.
used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->map.size() + (*r)[i].treefile->map.size();
used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->off + (*r)[i].treefile->off;
}
static int warned = 0;
@@ -1840,15 +1840,48 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
std::atomic<long long> poolpos(0);
for (size_t i = 0; i < CPUS; i++) {
if (readers[i].poolfile->map.size() > 0) {
if (fwrite(readers[i].poolfile->map.c_str(), readers[i].poolfile->map.size(), 1, poolfile) != 1) {
perror("Reunify string pool");
// If the memfile is not done yet, it is in memory, so just copy the memory.
// Otherwise, we need to merge memory and file.
if (readers[i].poolfile->fp == NULL) {
// still in memory
if (readers[i].poolfile->map.size() > 0) {
if (fwrite(readers[i].poolfile->map.c_str(), readers[i].poolfile->map.size(), 1, poolfile) != 1) {
perror("Reunify string pool");
exit(EXIT_WRITE);
}
}
pool_off[i] = poolpos;
poolpos += readers[i].poolfile->map.size();
} else {
// split into memory and file
if (fflush(readers[i].poolfile->fp) != 0) {
perror("fflush poolfile");
exit(EXIT_WRITE);
}
char *s = (char *) mmap(NULL, readers[i].poolfile->off, PROT_READ, MAP_PRIVATE, readers[i].poolfile->fd, 0);
if (s == MAP_FAILED) {
perror("mmap string pool for copy");
exit(EXIT_MEMORY);
}
madvise(s, readers[i].poolfile->off, MADV_SEQUENTIAL);
if (fwrite(s, sizeof(char), readers[i].poolfile->off, poolfile) != readers[i].poolfile->off) {
perror("Reunify string pool (split)");
exit(EXIT_WRITE);
}
if (munmap(s, readers[i].poolfile->off) != 0) {
perror("unmap string pool for copy");
exit(EXIT_MEMORY);
}
pool_off[i] = poolpos;
poolpos += readers[i].poolfile->off;
}
pool_off[i] = poolpos;
poolpos += readers[i].poolfile->map.size();
memfile_close(readers[i].poolfile);
}
memfile.cpp
+52 -6
View File
@@ -19,17 +19,27 @@ struct memfile *memfile_open(int fd) {
mf->fd = fd;
mf->tree = 0;
mf->off = 0;
return mf;
}
int memfile_close(struct memfile *file) {
if (write(file->fd, file->map.c_str(), file->map.size()) != (ssize_t) file->map.size()) {
return -1;
}
// If it isn't full yet, flush out the string to the file now.
// If it is full, close out the buffered file writer.
if (file->fd >= 0) {
if (close(file->fd) != 0) {
if (file->fp == NULL) {
if (write(file->fd, file->map.c_str(), file->map.size()) != (ssize_t) file->map.size()) {
return -1;
}
if (file->fd >= 0) {
if (close(file->fd) != 0) {
return -1;
}
}
} else {
if (fclose(file->fp) != 0) {
return -1;
}
}
@@ -39,6 +49,42 @@ int memfile_close(struct memfile *file) {
}
int memfile_write(struct memfile *file, void *s, long long len) {
file->map.append(std::string((char *) s, len));
// If it is full, append to the file.
// If it is not full yet, append to the string in memory.
if (file->fp != NULL) {
if (fwrite(s, sizeof(char), len, file->fp) != (size_t) len) {
return 0;
}
file->off += len;
} else {
file->map.append(std::string((char *) s, len));
file->off += len;
}
return len;
}
void memfile_full(struct memfile *file) {
// The file is full. Write out a copy of whatever has accumulated in memory
// to the file, and switch to appending to the file. Existing references
// into the memory still work.
if (file->fp != NULL) {
fprintf(stderr, "memfile marked full twice\n");
exit(EXIT_IMPOSSIBLE);
}
fprintf(stderr, "Tippecanoe string pool is full (%zu); switching to file\n", file->off);
file->fp = fdopen(file->fd, "wb");
if (file->fp == NULL) {
fprintf(stderr, "fdopen memfile: %s\n", strerror(errno));
exit(EXIT_OPEN);
}
if (fwrite(file->map.c_str(), sizeof(char), file->map.size(), file->fp) != file->map.size()) {
fprintf(stderr, "memfile write: %s\n", strerror(errno));
exit(EXIT_WRITE);
}
}
memfile.hpp
+4
View File
@@ -3,15 +3,19 @@
#include <atomic>
#include <string>
#include "errors.hpp"
struct memfile {
int fd = 0;
std::string map;
unsigned long tree = 0;
FILE *fp = NULL;
size_t off = 0;
};
struct memfile *memfile_open(int fd);
int memfile_close(struct memfile *file);
int memfile_write(struct memfile *file, void *s, long long len);
void memfile_full(struct memfile *file);
#endif
pool.cpp
+21 -9
View File
@@ -37,7 +37,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
size_t depth = 0;
// In typical data, traversal depth generally stays under 2.5x
size_t max = 3 * log(treefile->map.size() / sizeof(struct stringpool)) / log(2);
size_t max = 3 * log(treefile->off / sizeof(struct stringpool)) / log(2);
if (max < 30) {
max = 30;
}
@@ -61,8 +61,10 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
if (depth > max) {
// Search is very deep, so string is probably unique.
// Add it to the pool without adding it to the search tree.
// This might go either to memory or the file, depending on whether
// the pool is full yet.
long long off = poolfile->map.size();
long long off = poolfile->off;
if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write");
exit(EXIT_WRITE);
@@ -75,12 +77,22 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
}
}
if ((size_t) (poolfile->map.size() + treefile->map.size()) > memsize / CPUS / 2) {
// Size of memory divided by 5 from observation of OOM errors on ECS
if ((size_t) (poolfile->off + treefile->off) > memsize / CPUS / 5) {
// If the pool and search tree get to be larger than physical memory,
// then searching will start thrashing. Just append the new string
// to the pool rather than letting the tree grow any further.
// then searching will start thrashing. Switch to appending strings
// to the file instead of keeping them in memory.
long long off = poolfile->map.size();
if (poolfile->fp == NULL) {
memfile_full(poolfile);
}
}
if (poolfile->fp != NULL) {
// We are now appending to the file, so don't try to keep tree references
// to the newly-added strings.
long long off = poolfile->off;
if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write");
exit(EXIT_WRITE);
@@ -100,7 +112,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
ssp = ((char *) sp) - treefile->map.c_str();
}
long long off = poolfile->map.size();
long long off = poolfile->off;
if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write");
exit(EXIT_WRITE);
@@ -110,7 +122,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
exit(EXIT_WRITE);
}
if (off >= LONG_MAX || treefile->map.size() >= LONG_MAX) {
if (off >= LONG_MAX || treefile->off >= LONG_MAX) {
// Tree or pool is bigger than 2GB
static bool warned = false;
if (!warned) {
@@ -125,7 +137,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
tsp.right = 0;
tsp.off = off;
long long p = treefile->map.size();
long long p = treefile->off;
if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) {
perror("memfile write");
exit(EXIT_WRITE);