diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 16:35:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Move the string pool and search tree from mmap to allocated memory

Browse Source
This commit is contained in:
Erica Fischer
2023-01-12 11:19:44 -08:00
parent 42e2434cca
commit da2b5bd9fb
4 changed files with 22 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+4 -4
View File
@@ -116,7 +116,7 @@ void checkdisk(std::vector<struct reader> *r) {
for (size_t i = 0; i < r->size(); i++) {
// Pool and tree are used once.
// Geometry and index will be duplicated during sorting and tiling.
used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len;
used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->map.size() + (*r)[i].treefile->map.size();
}
static int warned = 0;
@@ -1847,15 +1847,15 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
std::atomic<long long> poolpos(0);
for (size_t i = 0; i < CPUS; i++) {
if (readers[i].poolfile->off > 0) {
if (fwrite(readers[i].poolfile->map, readers[i].poolfile->off, 1, poolfile) != 1) {
if (readers[i].poolfile->map.size() > 0) {
if (fwrite(readers[i].poolfile->map.c_str(), readers[i].poolfile->map.size(), 1, poolfile) != 1) {
perror("Reunify string pool");
exit(EXIT_WRITE);
}
}
pool_off[i] = poolpos;
poolpos += readers[i].poolfile->off;
poolpos += readers[i].poolfile->map.size();
memfile_close(readers[i].poolfile);
}
memfile.cpp
+2 -29
View File
@@ -12,28 +12,19 @@ struct memfile *memfile_open(int fd) {
return NULL;
}
char *map = (char *) mmap(NULL, INITIAL, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if (map == MAP_FAILED) {
return NULL;
}
struct memfile *mf = new memfile;
if (mf == NULL) {
munmap(map, INITIAL);
return NULL;
}
mf->fd = fd;
mf->map = map;
mf->len = INITIAL;
mf->off = 0;
mf->tree = 0;
return mf;
}
int memfile_close(struct memfile *file) {
if (munmap(file->map, file->len) != 0) {
if (write(file->fd, file->map.c_str(), file->map.size()) != (ssize_t) file->map.size()) {
return -1;
}
@@ -48,24 +39,6 @@ int memfile_close(struct memfile *file) {
}
int memfile_write(struct memfile *file, void *s, long long len) {
if (file->off + len > file->len) {
if (munmap(file->map, file->len) != 0) {
return -1;
}
file->len += (len + INCREMENT + 1) / INCREMENT * INCREMENT;
if (ftruncate(file->fd, file->len) != 0) {
return -1;
}
file->map = (char *) mmap(NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0);
if (file->map == MAP_FAILED) {
return -1;
}
}
memcpy(file->map + file->off, s, len);
file->off += len;
file->map.append(std::string((char *) s, len));
return len;
}
memfile.hpp
+2 -7
View File
@@ -2,17 +2,12 @@
#define MEMFILE_HPP
#include <atomic>
#include <string>
struct memfile {
int fd = 0;
char *map = NULL;
std::atomic<long long> len;
long long off = 0;
std::string map;
unsigned long tree = 0;
memfile()
: len(0) {
}
};
struct memfile *memfile_open(int fd);
pool.cpp
+14 -14
View File
@@ -37,24 +37,24 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
size_t depth = 0;
// In typical data, traversal depth generally stays under 2.5x
size_t max = 3 * log(treefile->off / sizeof(struct stringpool)) / log(2);
size_t max = 3 * log(treefile->map.size() / sizeof(struct stringpool)) / log(2);
if (max < 30) {
max = 30;
}
while (*sp != 0) {
int cmp = swizzlecmp(s, poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off + 1);
int cmp = swizzlecmp(s, poolfile->map.c_str() + ((struct stringpool *) (treefile->map.c_str() + *sp))->off + 1);
if (cmp == 0) {
cmp = type - (poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off)[0];
cmp = type - (poolfile->map.c_str() + ((struct stringpool *) (treefile->map.c_str() + *sp))->off)[0];
}
if (cmp < 0) {
sp = &(((struct stringpool *) (treefile->map + *sp))->left);
sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->left);
} else if (cmp > 0) {
sp = &(((struct stringpool *) (treefile->map + *sp))->right);
sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->right);
} else {
return ((struct stringpool *) (treefile->map + *sp))->off;
return ((struct stringpool *) (treefile->map.c_str() + *sp))->off;
}
depth++;
@@ -62,7 +62,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
// Search is very deep, so string is probably unique.
// Add it to the pool without adding it to the search tree.
long long off = poolfile->off;
long long off = poolfile->map.size();
if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write");
exit(EXIT_WRITE);
@@ -75,12 +75,12 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
}
}
if ((size_t) (poolfile->off + treefile->off) > memsize / CPUS / 2) {
if ((size_t) (poolfile->map.size() + treefile->map.size()) > memsize / CPUS / 2) {
// If the pool and search tree get to be larger than physical memory,
// then searching will start thrashing. Just append the new string
// to the pool rather than letting the tree grow any further.
long long off = poolfile->off;
long long off = poolfile->map.size();
if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write");
exit(EXIT_WRITE);
@@ -97,10 +97,10 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
if (sp == &treefile->tree) {
ssp = -1;
} else {
ssp = ((char *) sp) - treefile->map;
ssp = ((char *) sp) - treefile->map.c_str();
}
long long off = poolfile->off;
long long off = poolfile->map.size();
if (memfile_write(poolfile, &type, 1) < 0) {
perror("memfile write");
exit(EXIT_WRITE);
@@ -110,7 +110,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
exit(EXIT_WRITE);
}
if (off >= LONG_MAX || treefile->off >= LONG_MAX) {
if (off >= LONG_MAX || treefile->map.size() >= LONG_MAX) {
// Tree or pool is bigger than 2GB
static bool warned = false;
if (!warned) {
@@ -125,7 +125,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
tsp.right = 0;
tsp.off = off;
long long p = treefile->off;
long long p = treefile->map.size();
if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) {
perror("memfile write");
exit(EXIT_WRITE);
@@ -134,7 +134,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
if (ssp == -1) {
treefile->tree = p;
} else {
*((long long *) (treefile->map + ssp)) = p;
*((long long *) (treefile->map.c_str() + ssp)) = p;
}
return off;
}