From dcda6bcdd5e6a1e2c4e4c7afb94bc4ae1f9c8165 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Thu, 4 Apr 2024 15:29:45 -0700 Subject: [PATCH] Work in progress on a new feature sort --- serial.cpp | 12 ++-- sort.cpp | 204 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+), 7 deletions(-) diff --git a/serial.cpp b/serial.cpp index b7dc5ef7..90d21636 100644 --- a/serial.cpp +++ b/serial.cpp @@ -187,6 +187,8 @@ static void write_geometry(drawvec const &dv, std::string &out, long long wx, lo std::string serialize_feature(serial_feature *sf, long long wx, long long wy) { std::string s; + // index comes first so the sort can find it easily + serialize_index(s, sf->index); serialize_byte(s, sf->t); #define FLAG_LAYER 7 @@ -223,9 +225,6 @@ std::string serialize_feature(serial_feature *sf, long long wx, long long wy) { write_geometry(sf->geometry, s, wx, wy); - if (sf->index != 0) { - serialize_index(s, sf->index); - } if (sf->label_point != 0) { serialize_index(s, sf->label_point); } @@ -249,6 +248,9 @@ serial_feature deserialize_feature(std::string const &geoms, unsigned z, unsigne serial_feature sf; const char *cp = geoms.c_str(); + // index comes first so the sort can find it easily + deserialize_index(&cp, &sf.index); + deserialize_byte(&cp, &sf.t); deserialize_long_long(&cp, &sf.layer); @@ -272,15 +274,11 @@ serial_feature deserialize_feature(std::string const &geoms, unsigned z, unsigne deserialize_int(&cp, &sf.segment); - sf.index = 0; sf.label_point = 0; sf.extent = 0; sf.geometry = decode_geometry(&cp, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]); - if (sf.layer & (1 << FLAG_INDEX)) { - deserialize_index(&cp, &sf.index); - } if (sf.layer & (1 << FLAG_LABEL_POINT)) { deserialize_index(&cp, &sf.label_point); } diff --git a/sort.cpp b/sort.cpp index c5350c8b..3208aaf5 100644 --- a/sort.cpp +++ b/sort.cpp @@ -4,6 +4,10 @@ #include #include +#include "projection.hpp" +#include "errors.hpp" +#include "serial.hpp" + #define MAX_MEMORY (1024 * 1024 * 1024) // 1 GB void fqsort(std::vector &inputs, size_t width, int (*cmp)(const void *, const void *), FILE *out, size_t mem) { @@ -125,3 +129,203 @@ void fqsort(std::vector &inputs, size_t width, int (*cmp)(const void *, fqsort(v2, width, cmp, out, mem); fclose(fp2); } + +#if 0 + +struct indexed_feature { + std::string feature; + index_t index; + size_t seq; + + bool operator<(indexed_feature const &f) const { + if (index < f.index) { + return true; + } + if (index == f.index) { + if (seq < f.seq) { + return true; + } + } + return false; + } +}; + +int deserialize_ulong_long(FILE *fp, unsigned long long *zigzag) { + *zigzag = 0; + int shift = 0; + + while (true) { + char c; + if (fread(&c, sizeof(char), 1, fp) != 1) { + return 0; + } + + if ((c & 0x80) == 0) { + *zigzag |= ((unsigned long long) c) << shift; + shift += 7; + break; + } else { + *zigzag |= ((unsigned long long) (c & 0x7F)) << shift; + shift += 7; + } + } + + return 1; +} + +void feature_sort(std::vector &inputs, FILE *out, size_t mem) { + FILE *fp1, *fp2; + size_t seq = 0; + indexed_feature pivot; + + if (mem > MAX_MEMORY) { + mem = MAX_MEMORY; + } + + { + // read some elements into memory to choose a pivot from + // + // this is in its own scope so `buf` can go out of scope + // before trying to do any sub-sorts. + + std::vector buf; + size_t bufsize = 0; + + bool read_everything = false; + for (size_t i = 0; i < inputs.size(); i++) { + if (bufsize > mem) { + break; + } + + while (true) { + unsigned long long len; + + if (deserialize_ulong_long(inputs[i], &len) == 0) { + if (i + 1 == inputs.size()) { + read_everything = true; + } + break; + } + + indexed_feature f; + f.feature.resize(len); + if (fread((char *) f.feature.c_str(), len, sizeof(char), inputs[i]) != 1) { + perror("fread"); + exit(EXIT_READ); + } + + const char *cp = f.feature.c_str(); + deserialize_ulong_long(&cp, &f.index); + f.seq = seq++; + + buf.push_back(std::move(f)); + bufsize += len; + if (bufsize > mem) { + break; + } + } + } + + std::stable_sort(buf.begin(), buf.end()); + + // If that was everything we have to sort, we are done. + + if (read_everything) { + std::atomic fpos(0); + for (auto const &f : buf) { + serialize_ulong_long(out, f.feature.size(), &fpos, "sort output"); + fwrite_check(f.feature.c_str(), f.feature.size(), 1, out, &fpos, "sort output"); + } + return; + } + + // Otherwise, choose a pivot from it, make some temporary files, + // write what we have to those files, and then partition the rest + // of the input into them. + + // This would be unstable if the pivot is one of several elements + // that compare equal. Does it matter? + + size_t pivot_off = buf.size() / 2; + pivot = buf[pivot_off]; + + std::string t1 = "/tmp/sort1.XXXXXX"; + std::string t2 = "/tmp/sort2.XXXXXX"; + + int fd1 = mkstemp((char *) t1.c_str()); + unlink(t1.c_str()); + int fd2 = mkstemp((char *) t2.c_str()); + unlink(t2.c_str()); + + fp1 = fdopen(fd1, "w+b"); + if (fp1 == NULL) { + perror(t1.c_str()); + exit(EXIT_FAILURE); + } + fp2 = fdopen(fd2, "w+b"); + if (fp2 == NULL) { + perror(t2.c_str()); + exit(EXIT_FAILURE); + } + + std::atomic fpos(0); + for (size_t i = 0; i < pivot_off; i++) { + serialize_ulong_long(fp1, buf[i].feature.size(), &fpos, "sort output"); + fwrite_check(buf[i].feature.c_str(), buf[i].feature.size(), 1, fp1, &fpos, "sort output"); + } + + for (size_t i = pivot_off; i < buf.size(); i++) { + serialize_ulong_long(fp2, buf[i].feature.size(), &fpos, "sort output"); + fwrite_check(buf[i].feature.c_str(), buf[i].feature.size(), 1, fp2, &fpos, "sort output"); + } + } + + // read the remaining input into the temporary files + + std::atomic fpos(0); + for (size_t i = 0; i < inputs.size(); i++) { + while (true) { + unsigned long long len; + + if (deserialize_ulong_long(inputs[i], &len) == 0) { + break; + } + + indexed_feature f; + f.feature.resize(len); + if (fread((char *) f.feature.c_str(), len, sizeof(char), inputs[i]) != 1) { + perror("fread"); + exit(EXIT_READ); + } + + const char *cp = f.feature.c_str(); + deserialize_ulong_long(&cp, &f.index); + f.seq = seq++; + + if (f < pivot) { + serialize_ulong_long(fp1, f.feature.size(), &fpos, "sort output"); + fwrite_check(f.feature.c_str(), f.feature.size(), 1, fp1, &fpos, "sort output"); + } else { + serialize_ulong_long(fp2, f.feature.size(), &fpos, "sort output"); + fwrite_check(f.feature.c_str(), f.feature.size(), 1, fp2, &fpos, "sort output"); + } + } + } + + // Now sort the sub-ranges into the output. + + rewind(fp1); + rewind(fp2); + + std::vector v1; + v1.emplace_back(fp1); + feature_sort(v1, out, mem); + fclose(fp1); + + std::vector v2; + v2.emplace_back(fp2); + feature_sort(v2, out, mem); + fclose(fp2); +} + +#endif