diff --git a/Makefile b/Makefile index 9850499c..cff204b8 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ tippecanoe-enumerate: enumerate.o tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o +tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o diff --git a/main.cpp b/main.cpp index 28f98409..898c1030 100644 --- a/main.cpp +++ b/main.cpp @@ -105,6 +105,7 @@ struct source { size_t CPUS; size_t TEMP_FILES; long long MAX_FILES; +size_t memsize; static long long diskfree; char **av; @@ -1031,17 +1032,8 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d } } -void radix(std::vector &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic *geompos, int maxzoom, int basezoom, double droprate, double gamma) { - // Run through the index and geometry for each reader, - // splitting the contents out by index into as many - // sub-files as we can write to simultaneously. - - // Then sort each of those by index, recursively if it is - // too big to fit in memory. - - // Then concatenate each of the sub-outputs into a final output. - - long long mem; +static size_t calc_memsize() { + size_t mem; #ifdef __APPLE__ int64_t hw_memsize; @@ -1062,6 +1054,21 @@ void radix(std::vector &readers, int nreaders, FILE *geomfile, FI mem = (long long) pages * pagesize; #endif + return mem; +} + +void radix(std::vector &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic *geompos, int maxzoom, int basezoom, double droprate, double gamma) { + // Run through the index and geometry for each reader, + // splitting the contents out by index into as many + // sub-files as we can write to simultaneously. + + // Then sort each of those by index, recursively if it is + // too big to fit in memory. + + // Then concatenate each of the sub-outputs into a final output. + + long long mem = memsize; + // Just for code coverage testing. Deeply recursive sorting is very slow // compared to sorting in memory. if (additional[A_PREFER_RADIX_SORT]) { @@ -2650,6 +2657,8 @@ int main(int argc, char **argv) { int files_open_at_start; json_object *filter = NULL; + memsize = calc_memsize(); + for (i = 0; i < 256; i++) { prevent[i] = 0; additional[i] = 0; diff --git a/main.hpp b/main.hpp index 60bd36b6..815bd24c 100644 --- a/main.hpp +++ b/main.hpp @@ -47,6 +47,7 @@ extern int extra_detail; extern size_t CPUS; extern size_t TEMP_FILES; +extern size_t memsize; extern size_t max_tile_size; extern size_t max_tile_features; diff --git a/pool.cpp b/pool.cpp index 71de62f5..d821caa6 100644 --- a/pool.cpp +++ b/pool.cpp @@ -3,6 +3,7 @@ #include #include #include +#include "main.hpp" #include "memfile.hpp" #include "pool.hpp" #include "errors.hpp" @@ -74,6 +75,23 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char } } + if ((size_t) (poolfile->off + treefile->off) > memsize / CPUS / 2) { + // If the pool and search tree get to be larger than physical memory, + // then searching will start thrashing. Just append the new string + // to the pool rather than letting the tree grow any further. + + long long off = poolfile->off; + if (memfile_write(poolfile, &type, 1) < 0) { + perror("memfile write"); + exit(EXIT_WRITE); + } + if (memfile_write(poolfile, (void *) s, strlen(s) + 1) < 0) { + perror("memfile write"); + exit(EXIT_WRITE); + } + return off; + } + // *sp is probably in the memory-mapped file, and will move if the file grows. long long ssp; if (sp == &treefile->tree) { diff --git a/tile-join.cpp b/tile-join.cpp index f72c059d..82eee89d 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -25,7 +25,6 @@ #include #include "mvt.hpp" #include "projection.hpp" -#include "pool.hpp" #include "mbtiles.hpp" #include "geometry.hpp" #include "dirtiles.hpp"