From da2b5bd9fbfdbd9498d7b5f3bf1dd6d977f8ac49 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Thu, 12 Jan 2023 11:19:44 -0800 Subject: [PATCH] Move the string pool and search tree from mmap to allocated memory --- main.cpp | 8 ++++---- memfile.cpp | 31 ++----------------------------- memfile.hpp | 9 ++------- pool.cpp | 28 ++++++++++++++-------------- 4 files changed, 22 insertions(+), 54 deletions(-) diff --git a/main.cpp b/main.cpp index 898c1030..d3e5d740 100644 --- a/main.cpp +++ b/main.cpp @@ -116,7 +116,7 @@ void checkdisk(std::vector *r) { for (size_t i = 0; i < r->size(); i++) { // Pool and tree are used once. // Geometry and index will be duplicated during sorting and tiling. - used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len; + used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->map.size() + (*r)[i].treefile->map.size(); } static int warned = 0; @@ -1847,15 +1847,15 @@ std::pair read_input(std::vector &sources, char *fname, i std::atomic poolpos(0); for (size_t i = 0; i < CPUS; i++) { - if (readers[i].poolfile->off > 0) { - if (fwrite(readers[i].poolfile->map, readers[i].poolfile->off, 1, poolfile) != 1) { + if (readers[i].poolfile->map.size() > 0) { + if (fwrite(readers[i].poolfile->map.c_str(), readers[i].poolfile->map.size(), 1, poolfile) != 1) { perror("Reunify string pool"); exit(EXIT_WRITE); } } pool_off[i] = poolpos; - poolpos += readers[i].poolfile->off; + poolpos += readers[i].poolfile->map.size(); memfile_close(readers[i].poolfile); } diff --git a/memfile.cpp b/memfile.cpp index 77508367..3f212a56 100644 --- a/memfile.cpp +++ b/memfile.cpp @@ -12,28 +12,19 @@ struct memfile *memfile_open(int fd) { return NULL; } - char *map = (char *) mmap(NULL, INITIAL, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (map == MAP_FAILED) { - return NULL; - } - struct memfile *mf = new memfile; if (mf == NULL) { - munmap(map, INITIAL); return NULL; } mf->fd = fd; - mf->map = map; - mf->len = INITIAL; - mf->off = 0; mf->tree = 0; return mf; } int memfile_close(struct memfile *file) { - if (munmap(file->map, file->len) != 0) { + if (write(file->fd, file->map.c_str(), file->map.size()) != (ssize_t) file->map.size()) { return -1; } @@ -48,24 +39,6 @@ int memfile_close(struct memfile *file) { } int memfile_write(struct memfile *file, void *s, long long len) { - if (file->off + len > file->len) { - if (munmap(file->map, file->len) != 0) { - return -1; - } - - file->len += (len + INCREMENT + 1) / INCREMENT * INCREMENT; - - if (ftruncate(file->fd, file->len) != 0) { - return -1; - } - - file->map = (char *) mmap(NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0); - if (file->map == MAP_FAILED) { - return -1; - } - } - - memcpy(file->map + file->off, s, len); - file->off += len; + file->map.append(std::string((char *) s, len)); return len; } diff --git a/memfile.hpp b/memfile.hpp index 185544f4..0baf3a53 100644 --- a/memfile.hpp +++ b/memfile.hpp @@ -2,17 +2,12 @@ #define MEMFILE_HPP #include +#include struct memfile { int fd = 0; - char *map = NULL; - std::atomic len; - long long off = 0; + std::string map; unsigned long tree = 0; - - memfile() - : len(0) { - } }; struct memfile *memfile_open(int fd); diff --git a/pool.cpp b/pool.cpp index d821caa6..c0003d87 100644 --- a/pool.cpp +++ b/pool.cpp @@ -37,24 +37,24 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char size_t depth = 0; // In typical data, traversal depth generally stays under 2.5x - size_t max = 3 * log(treefile->off / sizeof(struct stringpool)) / log(2); + size_t max = 3 * log(treefile->map.size() / sizeof(struct stringpool)) / log(2); if (max < 30) { max = 30; } while (*sp != 0) { - int cmp = swizzlecmp(s, poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off + 1); + int cmp = swizzlecmp(s, poolfile->map.c_str() + ((struct stringpool *) (treefile->map.c_str() + *sp))->off + 1); if (cmp == 0) { - cmp = type - (poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off)[0]; + cmp = type - (poolfile->map.c_str() + ((struct stringpool *) (treefile->map.c_str() + *sp))->off)[0]; } if (cmp < 0) { - sp = &(((struct stringpool *) (treefile->map + *sp))->left); + sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->left); } else if (cmp > 0) { - sp = &(((struct stringpool *) (treefile->map + *sp))->right); + sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->right); } else { - return ((struct stringpool *) (treefile->map + *sp))->off; + return ((struct stringpool *) (treefile->map.c_str() + *sp))->off; } depth++; @@ -62,7 +62,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char // Search is very deep, so string is probably unique. // Add it to the pool without adding it to the search tree. - long long off = poolfile->off; + long long off = poolfile->map.size(); if (memfile_write(poolfile, &type, 1) < 0) { perror("memfile write"); exit(EXIT_WRITE); @@ -75,12 +75,12 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char } } - if ((size_t) (poolfile->off + treefile->off) > memsize / CPUS / 2) { + if ((size_t) (poolfile->map.size() + treefile->map.size()) > memsize / CPUS / 2) { // If the pool and search tree get to be larger than physical memory, // then searching will start thrashing. Just append the new string // to the pool rather than letting the tree grow any further. - long long off = poolfile->off; + long long off = poolfile->map.size(); if (memfile_write(poolfile, &type, 1) < 0) { perror("memfile write"); exit(EXIT_WRITE); @@ -97,10 +97,10 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char if (sp == &treefile->tree) { ssp = -1; } else { - ssp = ((char *) sp) - treefile->map; + ssp = ((char *) sp) - treefile->map.c_str(); } - long long off = poolfile->off; + long long off = poolfile->map.size(); if (memfile_write(poolfile, &type, 1) < 0) { perror("memfile write"); exit(EXIT_WRITE); @@ -110,7 +110,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char exit(EXIT_WRITE); } - if (off >= LONG_MAX || treefile->off >= LONG_MAX) { + if (off >= LONG_MAX || treefile->map.size() >= LONG_MAX) { // Tree or pool is bigger than 2GB static bool warned = false; if (!warned) { @@ -125,7 +125,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char tsp.right = 0; tsp.off = off; - long long p = treefile->off; + long long p = treefile->map.size(); if (memfile_write(treefile, &tsp, sizeof(struct stringpool)) < 0) { perror("memfile write"); exit(EXIT_WRITE); @@ -134,7 +134,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char if (ssp == -1) { treefile->tree = p; } else { - *((long long *) (treefile->map + ssp)) = p; + *((long long *) (treefile->map.c_str() + ssp)) = p; } return off; }