mirror of
https://github.com/felt/tippecanoe.git
synced 2026-10-02 08:25:40 +02:00
Work in progress on a new feature sort
This commit is contained in:
Erica Fischer
+5
-7
@@ -187,6 +187,8 @@ static void write_geometry(drawvec const &dv, std::string &out, long long wx, lo
|
||||
std::string serialize_feature(serial_feature *sf, long long wx, long long wy) {
|
||||
std::string s;
|
||||
|
||||
// index comes first so the sort can find it easily
|
||||
serialize_index(s, sf->index);
|
||||
serialize_byte(s, sf->t);
|
||||
|
||||
#define FLAG_LAYER 7
|
||||
@@ -223,9 +225,6 @@ std::string serialize_feature(serial_feature *sf, long long wx, long long wy) {
|
||||
|
||||
write_geometry(sf->geometry, s, wx, wy);
|
||||
|
||||
if (sf->index != 0) {
|
||||
serialize_index(s, sf->index);
|
||||
}
|
||||
if (sf->label_point != 0) {
|
||||
serialize_index(s, sf->label_point);
|
||||
}
|
||||
@@ -249,6 +248,9 @@ serial_feature deserialize_feature(std::string const &geoms, unsigned z, unsigne
|
||||
serial_feature sf;
|
||||
const char *cp = geoms.c_str();
|
||||
|
||||
// index comes first so the sort can find it easily
|
||||
deserialize_index(&cp, &sf.index);
|
||||
|
||||
deserialize_byte(&cp, &sf.t);
|
||||
deserialize_long_long(&cp, &sf.layer);
|
||||
|
||||
@@ -272,15 +274,11 @@ serial_feature deserialize_feature(std::string const &geoms, unsigned z, unsigne
|
||||
|
||||
deserialize_int(&cp, &sf.segment);
|
||||
|
||||
sf.index = 0;
|
||||
sf.label_point = 0;
|
||||
sf.extent = 0;
|
||||
|
||||
sf.geometry = decode_geometry(&cp, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]);
|
||||
|
||||
if (sf.layer & (1 << FLAG_INDEX)) {
|
||||
deserialize_index(&cp, &sf.index);
|
||||
}
|
||||
if (sf.layer & (1 << FLAG_LABEL_POINT)) {
|
||||
deserialize_index(&cp, &sf.label_point);
|
||||
}
|
||||
|
||||
@@ -4,6 +4,10 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
#include "projection.hpp"
|
||||
#include "errors.hpp"
|
||||
#include "serial.hpp"
|
||||
|
||||
#define MAX_MEMORY (1024 * 1024 * 1024) // 1 GB
|
||||
|
||||
void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *, const void *), FILE *out, size_t mem) {
|
||||
@@ -125,3 +129,203 @@ void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *,
|
||||
fqsort(v2, width, cmp, out, mem);
|
||||
fclose(fp2);
|
||||
}
|
||||
|
||||
#if 0
|
||||
|
||||
struct indexed_feature {
|
||||
std::string feature;
|
||||
index_t index;
|
||||
size_t seq;
|
||||
|
||||
bool operator<(indexed_feature const &f) const {
|
||||
if (index < f.index) {
|
||||
return true;
|
||||
}
|
||||
if (index == f.index) {
|
||||
if (seq < f.seq) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
int deserialize_ulong_long(FILE *fp, unsigned long long *zigzag) {
|
||||
*zigzag = 0;
|
||||
int shift = 0;
|
||||
|
||||
while (true) {
|
||||
char c;
|
||||
if (fread(&c, sizeof(char), 1, fp) != 1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ((c & 0x80) == 0) {
|
||||
*zigzag |= ((unsigned long long) c) << shift;
|
||||
shift += 7;
|
||||
break;
|
||||
} else {
|
||||
*zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
|
||||
shift += 7;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
void feature_sort(std::vector<FILE *> &inputs, FILE *out, size_t mem) {
|
||||
FILE *fp1, *fp2;
|
||||
size_t seq = 0;
|
||||
indexed_feature pivot;
|
||||
|
||||
if (mem > MAX_MEMORY) {
|
||||
mem = MAX_MEMORY;
|
||||
}
|
||||
|
||||
{
|
||||
// read some elements into memory to choose a pivot from
|
||||
//
|
||||
// this is in its own scope so `buf` can go out of scope
|
||||
// before trying to do any sub-sorts.
|
||||
|
||||
std::vector<indexed_feature> buf;
|
||||
size_t bufsize = 0;
|
||||
|
||||
bool read_everything = false;
|
||||
for (size_t i = 0; i < inputs.size(); i++) {
|
||||
if (bufsize > mem) {
|
||||
break;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
unsigned long long len;
|
||||
|
||||
if (deserialize_ulong_long(inputs[i], &len) == 0) {
|
||||
if (i + 1 == inputs.size()) {
|
||||
read_everything = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
indexed_feature f;
|
||||
f.feature.resize(len);
|
||||
if (fread((char *) f.feature.c_str(), len, sizeof(char), inputs[i]) != 1) {
|
||||
perror("fread");
|
||||
exit(EXIT_READ);
|
||||
}
|
||||
|
||||
const char *cp = f.feature.c_str();
|
||||
deserialize_ulong_long(&cp, &f.index);
|
||||
f.seq = seq++;
|
||||
|
||||
buf.push_back(std::move(f));
|
||||
bufsize += len;
|
||||
if (bufsize > mem) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::stable_sort(buf.begin(), buf.end());
|
||||
|
||||
// If that was everything we have to sort, we are done.
|
||||
|
||||
if (read_everything) {
|
||||
std::atomic<long long> fpos(0);
|
||||
for (auto const &f : buf) {
|
||||
serialize_ulong_long(out, f.feature.size(), &fpos, "sort output");
|
||||
fwrite_check(f.feature.c_str(), f.feature.size(), 1, out, &fpos, "sort output");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, choose a pivot from it, make some temporary files,
|
||||
// write what we have to those files, and then partition the rest
|
||||
// of the input into them.
|
||||
|
||||
// This would be unstable if the pivot is one of several elements
|
||||
// that compare equal. Does it matter?
|
||||
|
||||
size_t pivot_off = buf.size() / 2;
|
||||
pivot = buf[pivot_off];
|
||||
|
||||
std::string t1 = "/tmp/sort1.XXXXXX";
|
||||
std::string t2 = "/tmp/sort2.XXXXXX";
|
||||
|
||||
int fd1 = mkstemp((char *) t1.c_str());
|
||||
unlink(t1.c_str());
|
||||
int fd2 = mkstemp((char *) t2.c_str());
|
||||
unlink(t2.c_str());
|
||||
|
||||
fp1 = fdopen(fd1, "w+b");
|
||||
if (fp1 == NULL) {
|
||||
perror(t1.c_str());
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
fp2 = fdopen(fd2, "w+b");
|
||||
if (fp2 == NULL) {
|
||||
perror(t2.c_str());
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
std::atomic<long long> fpos(0);
|
||||
for (size_t i = 0; i < pivot_off; i++) {
|
||||
serialize_ulong_long(fp1, buf[i].feature.size(), &fpos, "sort output");
|
||||
fwrite_check(buf[i].feature.c_str(), buf[i].feature.size(), 1, fp1, &fpos, "sort output");
|
||||
}
|
||||
|
||||
for (size_t i = pivot_off; i < buf.size(); i++) {
|
||||
serialize_ulong_long(fp2, buf[i].feature.size(), &fpos, "sort output");
|
||||
fwrite_check(buf[i].feature.c_str(), buf[i].feature.size(), 1, fp2, &fpos, "sort output");
|
||||
}
|
||||
}
|
||||
|
||||
// read the remaining input into the temporary files
|
||||
|
||||
std::atomic<long long> fpos(0);
|
||||
for (size_t i = 0; i < inputs.size(); i++) {
|
||||
while (true) {
|
||||
unsigned long long len;
|
||||
|
||||
if (deserialize_ulong_long(inputs[i], &len) == 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
indexed_feature f;
|
||||
f.feature.resize(len);
|
||||
if (fread((char *) f.feature.c_str(), len, sizeof(char), inputs[i]) != 1) {
|
||||
perror("fread");
|
||||
exit(EXIT_READ);
|
||||
}
|
||||
|
||||
const char *cp = f.feature.c_str();
|
||||
deserialize_ulong_long(&cp, &f.index);
|
||||
f.seq = seq++;
|
||||
|
||||
if (f < pivot) {
|
||||
serialize_ulong_long(fp1, f.feature.size(), &fpos, "sort output");
|
||||
fwrite_check(f.feature.c_str(), f.feature.size(), 1, fp1, &fpos, "sort output");
|
||||
} else {
|
||||
serialize_ulong_long(fp2, f.feature.size(), &fpos, "sort output");
|
||||
fwrite_check(f.feature.c_str(), f.feature.size(), 1, fp2, &fpos, "sort output");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Now sort the sub-ranges into the output.
|
||||
|
||||
rewind(fp1);
|
||||
rewind(fp2);
|
||||
|
||||
std::vector<FILE *> v1;
|
||||
v1.emplace_back(fp1);
|
||||
feature_sort(v1, out, mem);
|
||||
fclose(fp1);
|
||||
|
||||
std::vector<FILE *> v2;
|
||||
v2.emplace_back(fp2);
|
||||
feature_sort(v2, out, mem);
|
||||
fclose(fp2);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user