diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-04 17:35:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Move the string pool and search tree from mmap to allocated memory

Browse Source
This commit is contained in:
Erica Fischer
2023-01-12 11:19:44 -08:00
parent 42e2434cca
commit da2b5bd9fb
4 changed files with 22 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+4 -4
View File
@@ -116,7 +116,7 @@ void checkdisk(std::vector<struct reader> *r) {
for (size_t i = 0; i < r->size(); i++) { for (size_t i = 0; i < r->size(); i++) {
// Pool and tree are used once. // Pool and tree are used once.
// Geometry and index will be duplicated during sorting and tiling. // Geometry and index will be duplicated during sorting and tiling.
used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len; used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->map.size() + (*r)[i].treefile->map.size();
} }
static int warned = 0; static int warned = 0;
@@ -1847,15 +1847,15 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
std::atomic<long long> poolpos(0); std::atomic<long long> poolpos(0);
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
if (readers[i].poolfile->off > 0) { if (readers[i].poolfile->map.size() > 0) {
if (fwrite(readers[i].poolfile->map, readers[i].poolfile->off, 1, poolfile) != 1) { if (fwrite(readers[i].poolfile->map.c_str(), readers[i].poolfile->map.size(), 1, poolfile) != 1) {
perror("Reunify string pool"); perror("Reunify string pool");
exit(EXIT_WRITE); exit(EXIT_WRITE);
} }
} }
pool_off[i] = poolpos; pool_off[i] = poolpos;
poolpos += readers[i].poolfile->off; poolpos += readers[i].poolfile->map.size();
memfile_close(readers[i].poolfile); memfile_close(readers[i].poolfile);
} }
memfile.cpp
+2 -29
View File
@@ -12,28 +12,19 @@ struct memfile *memfile_open(int fd) {
return NULL; return NULL;
} }
char *map = (char *) mmap(NULL, INITIAL, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
return NULL;
}
struct memfile *mf = new memfile; struct memfile *mf = new memfile;
if (mf == NULL) { if (mf == NULL) {
munmap(map, INITIAL);
return NULL; return NULL;
} }
mf->fd = fd; mf->fd = fd;
mf->map = map;
mf->len = INITIAL;
mf->off = 0;
mf->tree = 0; mf->tree = 0;
return mf; return mf;
} }
int memfile_close(struct memfile *file) { int memfile_close(struct memfile *file) {
if (munmap(file->map, file->len) != 0) { if (write(file->fd, file->map.c_str(), file->map.size()) != (ssize_t) file->map.size()) {
return -1; return -1;
} }
@@ -48,24 +39,6 @@ int memfile_close(struct memfile *file) {
} }
int memfile_write(struct memfile *file, void *s, long long len) { int memfile_write(struct memfile *file, void *s, long long len) {
if (file->off + len > file->len) { file->map.append(std::string((char *) s, len));
if (munmap(file->map, file->len) != 0) {
return -1;
}
file->len += (len + INCREMENT + 1) / INCREMENT * INCREMENT;
if (ftruncate(file->fd, file->len) != 0) {
return -1;
}
file->map = (char *) mmap(NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0);
if (file->map == MAP_FAILED) {
return -1;
}
}
memcpy(file->map + file->off, s, len);
file->off += len;
return len; return len;
} }
memfile.hpp
+2 -7
View File
@@ -2,17 +2,12 @@
#define MEMFILE_HPP #define MEMFILE_HPP
#include <atomic> #include <atomic>
#include <string>
struct memfile { struct memfile {
int fd = 0; int fd = 0;
char *map = NULL; std::string map;
std::atomic<long long> len;
long long off = 0;
unsigned long tree = 0; unsigned long tree = 0;
memfile()
: len(0) {
}
}; };
struct memfile *memfile_open(int fd); struct memfile *memfile_open(int fd);
pool.cpp
+14 -14
View File
@@ -37,24 +37,24 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
size_t depth = 0; size_t depth = 0;
// In typical data, traversal depth generally stays under 2.5x // In typical data, traversal depth generally stays under 2.5x
size_t max = 3 * log(treefile->off / sizeof(struct stringpool)) / log(2); size_t max = 3 * log(treefile->map.size() / sizeof(struct stringpool)) / log(2);
if (max < 30) { if (max < 30) {
max = 30; max = 30;
} }
while (*sp != 0) { while (*sp != 0) {
int cmp = swizzlecmp(s, poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off + 1); int cmp = swizzlecmp(s, poolfile->map.c_str() + ((struct stringpool *) (treefile->map.c_str() + *sp))->off + 1);
if (cmp == 0) { if (cmp == 0) {
cmp = type - (poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off)[0]; cmp = type - (poolfile->map.c_str() + ((struct stringpool *) (treefile->map.c_str() + *sp))->off)[0];
} }
if (cmp < 0) { if (cmp < 0) {
sp = &(((struct stringpool *) (treefile->map + *sp))->left); sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->left);
} else if (cmp > 0) { } else if (cmp > 0) {
sp = &(((struct stringpool *) (treefile->map + *sp))->right); sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->right);
} else { } else {
return ((struct stringpool *) (treefile->map + *sp))->off; return ((struct stringpool *) (treefile->map.c_str() + *sp))->off;
} }
depth++; depth++;
@@ -62,7 +62,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
// Search is very deep, so string is probably unique. // Search is very deep, so string is probably unique.
// Add it to the pool without adding it to the search tree. // Add it to the pool without adding it to the search tree.
long long off = poolfile->off; long long off = poolfile->map.size();
if (memfile_write(poolfile, &type, 1) < 0) { if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write"); perror("memfile write");
exit(EXIT_WRITE); exit(EXIT_WRITE);
@@ -75,12 +75,12 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
} }
} }
if ((size_t) (poolfile->off + treefile->off) > memsize / CPUS / 2) { if ((size_t) (poolfile->map.size() + treefile->map.size()) > memsize / CPUS / 2) {
// If the pool and search tree get to be larger than physical memory, // If the pool and search tree get to be larger than physical memory,
// then searching will start thrashing. Just append the new string // then searching will start thrashing. Just append the new string
// to the pool rather than letting the tree grow any further. // to the pool rather than letting the tree grow any further.
long long off = poolfile->off; long long off = poolfile->map.size();
if (memfile_write(poolfile, &type, 1) < 0) { if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write"); perror("memfile write");
exit(EXIT_WRITE); exit(EXIT_WRITE);
@@ -97,10 +97,10 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
if (sp == &treefile->tree) { if (sp == &treefile->tree) {
ssp = -1; ssp = -1;
} else { } else {
ssp = ((char *) sp) - treefile->map; ssp = ((char *) sp) - treefile->map.c_str();
} }
long long off = poolfile->off; long long off = poolfile->map.size();
if (memfile_write(poolfile, &type, 1) < 0) { if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write"); perror("memfile write");
exit(EXIT_WRITE); exit(EXIT_WRITE);
@@ -110,7 +110,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
exit(EXIT_WRITE); exit(EXIT_WRITE);
} }
if (off >= LONG_MAX || treefile->off >= LONG_MAX) { if (off >= LONG_MAX || treefile->map.size() >= LONG_MAX) {
// Tree or pool is bigger than 2GB // Tree or pool is bigger than 2GB
static bool warned = false; static bool warned = false;
if (!warned) { if (!warned) {
@@ -125,7 +125,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
tsp.right = 0; tsp.right = 0;
tsp.off = off; tsp.off = off;
long long p = treefile->off; long long p = treefile->map.size();
if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) { if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) {
perror("memfile write"); perror("memfile write");
exit(EXIT_WRITE); exit(EXIT_WRITE);
@@ -134,7 +134,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
if (ssp == -1) { if (ssp == -1) {
treefile->tree = p; treefile->tree = p;
} else { } else {
*((long long *) (treefile->map + ssp)) = p; *((long long *) (treefile->map.c_str() + ssp)) = p;
} }
return off; return off;
} }