diff --git a/main.cpp b/main.cpp index 1f8b8f83..16f47200 100644 --- a/main.cpp +++ b/main.cpp @@ -742,8 +742,16 @@ void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::ato } void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int splits, long long mem, const char *tmpdir, long long *availfiles, FILE *geomfile, FILE *indexfile, std::atomic *geompos_out, long long *progress, long long *progress_max, long long *progress_reported, int maxzoom, int basezoom, double droprate, double gamma, struct drop_state *ds) { - // Arranged as bits to facilitate subdividing again if a subdivided file is still huge - int splitbits = log(splits) / log(2); + // Arranged as bits to facilitate subdividing again if a subdivided file is still huge. + // + // There must be at least two buckets: with only one, each subdivision would + // consume no bits of the index, so it would never reach the maximum prefix + // that stops the recursion, and the shift below would be by the full width + // of the index, which is undefined. + int splitbits = 1; + if (splits > 1) { + splitbits = log(splits) / log(2); + } splits = 1 << splitbits; FILE *geomfiles[splits]; @@ -890,7 +898,14 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split } if (indexst.st_size > 0) { - if (indexst.st_size + geomst.st_size < mem) { + // Subdividing again would only make progress if the next level could + // split into at least two buckets, which it can't if there are no longer + // enough files left to do it with. In that case, sort in memory instead, + // even though this is more memory than we wanted to use at once, since + // it is the only way left to get this bucket sorted. + bool can_subdivide = *availfiles / 4 > 1; + + if (indexst.st_size + geomst.st_size < mem || !can_subdivide) { std::atomic indexpos(indexst.st_size); int bytes = sizeof(struct index);