diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-05 01:45:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Crashy work in progress toward compression

Browse Source
This commit is contained in:
Erica Fischer
2023-01-24 09:38:07 -08:00
parent 3addc57bfc
commit f1bffd9de0
4 changed files with 44 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
geometry.cpp
+4 -7
View File
@@ -24,7 +24,7 @@
static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax); static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax);
drawvec decode_geometry(FILE *meta, std::atomic<long long> *geompos, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y) { drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y) {
drawvec out; drawvec out;
bbox[0] = LLONG_MAX; bbox[0] = LLONG_MAX;
@@ -37,10 +37,7 @@ drawvec decode_geometry(FILE *meta, std::atomic<long long> *geompos, int z, unsi
while (1) { while (1) {
draw d; draw d;
if (!deserialize_byte_io(meta, &d.op, geompos)) { deserialize_byte(meta, &d.op);
fprintf(stderr, "Internal error: Unexpected end of file in geometry\n");
exit(EXIT_IMPOSSIBLE);
}
if (d.op == VT_END) { if (d.op == VT_END) {
break; break;
} }
@@ -48,8 +45,8 @@ drawvec decode_geometry(FILE *meta, std::atomic<long long> *geompos, int z, unsi
if (d.op == VT_MOVETO || d.op == VT_LINETO) { if (d.op == VT_MOVETO || d.op == VT_LINETO) {
long long dx, dy; long long dx, dy;
deserialize_long_long_io(meta, &dx, geompos); deserialize_long_long(meta, &dx);
deserialize_long_long_io(meta, &dy, geompos); deserialize_long_long(meta, &dy);
wx += dx * (1 << geometry_scale); wx += dx * (1 << geometry_scale);
wy += dy * (1 << geometry_scale); wy += dy * (1 << geometry_scale);
geometry.hpp
+1 -1
View File
@@ -58,7 +58,7 @@ struct draw {
typedef std::vector<draw> drawvec; typedef std::vector<draw> drawvec;
struct serial_feature; struct serial_feature;
drawvec decode_geometry(FILE *meta, std::atomic<long long> *geompos, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y); drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y);
void to_tile_scale(drawvec &geom, int z, int detail); void to_tile_scale(drawvec &geom, int z, int detail);
drawvec from_tile_scale(drawvec const &geom, int z, int detail); drawvec from_tile_scale(drawvec const &geom, int z, int detail);
drawvec remove_noop(drawvec geom, int type, int shift); drawvec remove_noop(drawvec geom, int type, int shift);
serial.cpp
+34 -19
View File
@@ -86,39 +86,36 @@ size_t fwrite_check(const void *ptr, size_t size, size_t nitems, std::string &st
return nitems; return nitems;
} }
void serialize_ulong_long(std::string &out, unsigned long long zigzag, std::atomic<long long> *fpos) { void serialize_ulong_long(std::string &out, unsigned long long zigzag) {
while (1) { while (1) {
unsigned char b = zigzag & 0x7F; unsigned char b = zigzag & 0x7F;
if ((zigzag >> 7) != 0) { if ((zigzag >> 7) != 0) {
b |= 0x80; b |= 0x80;
out += b; out += b;
*fpos += 1;
zigzag >>= 7; zigzag >>= 7;
} else { } else {
out += b; out += b;
*fpos += 1;
break; break;
} }
} }
} }
void serialize_long_long(std::string &out, long long n, std::atomic<long long> *fpos) { void serialize_long_long(std::string &out, long long n) {
unsigned long long zigzag = protozero::encode_zigzag64(n); unsigned long long zigzag = protozero::encode_zigzag64(n);
serialize_ulong_long(out, zigzag, fpos); serialize_ulong_long(out, zigzag);
} }
void serialize_int(std::string &out, int n, std::atomic<long long> *fpos) { void serialize_int(std::string &out, int n) {
serialize_long_long(out, n, fpos); serialize_long_long(out, n);
} }
void serialize_byte(std::string &out, signed char n, std::atomic<long long> *fpos) { void serialize_byte(std::string &out, signed char n) {
out += n; out += n;
*fpos += sizeof(signed char);
} }
void serialize_uint(std::string &out, unsigned n, std::atomic<long long> *fpos) { void serialize_uint(std::string &out, unsigned n) {
serialize_ulong_long(out, n, fpos); serialize_ulong_long(out, n);
} }
// read from memory // read from memory
@@ -221,18 +218,19 @@ int deserialize_byte_io(FILE *f, signed char *n, std::atomic<long long> *geompos
return 1; return 1;
} }
static void write_geometry(drawvec const &dv, std::atomic<long long> *fpos, FILE *out, const char *fname, long long wx, long long wy) { static void write_geometry(drawvec const &dv, std::string &out, long long wx, long long wy) {
for (size_t i = 0; i < dv.size(); i++) { for (size_t i = 0; i < dv.size(); i++) {
if (dv[i].op == VT_MOVETO || dv[i].op == VT_LINETO) { if (dv[i].op == VT_MOVETO || dv[i].op == VT_LINETO) {
serialize_byte(out, dv[i].op, fpos, fname); serialize_byte(out, dv[i].op);
serialize_long_long(out, dv[i].x - wx, fpos, fname); serialize_long_long(out, dv[i].x - wx);
serialize_long_long(out, dv[i].y - wy, fpos, fname); serialize_long_long(out, dv[i].y - wy);
wx = dv[i].x; wx = dv[i].x;
wy = dv[i].y; wy = dv[i].y;
} else { } else {
serialize_byte(out, dv[i].op, fpos, fname); serialize_byte(out, dv[i].op);
} }
} }
serialize_byte(out, VT_END);
} }
// called from generating the next zoom level // called from generating the next zoom level
@@ -275,8 +273,12 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long
serialize_int(geomfile, sf->segment, geompos, fname); serialize_int(geomfile, sf->segment, geompos, fname);
write_geometry(sf->geometry, geompos, geomfile, fname, wx, wy); std::string geom;
serialize_byte(geomfile, VT_END, geompos, fname); write_geometry(sf->geometry, geom, wx, wy);
serialize_ulong_long(geomfile, geom.size(), geompos, fname);
fwrite_check(geom.c_str(), sizeof(char), geom.size(), geomfile, fname);
geompos += geom.size();
if (sf->index != 0) { if (sf->index != 0) {
serialize_ulong_long(geomfile, sf->index, geompos, fname); serialize_ulong_long(geomfile, sf->index, geompos, fname);
} }
@@ -335,7 +337,20 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_
sf.label_point = 0; sf.label_point = 0;
sf.extent = 0; sf.extent = 0;
sf.geometry = decode_geometry(geoms, geompos_in, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]); unsigned long long geom_len;
deserialize_ulong_long_io(geoms, &geom_len, geompos_in);
std::string geom;
geom.resize(geom_len);
if (fread((void *) geom.c_str(), sizeof(char), geom_len, geoms) != geom_len) {
fprintf(stderr, "Short read (%llu) from geometry\n", geom_len);
exit(EXIT_READ);
}
*geompos_in += geom_len;
char *s = (char *) geom.c_str();
sf.geometry = decode_geometry(&s, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]);
if (sf.layer & (1 << FLAG_INDEX)) { if (sf.layer & (1 << FLAG_INDEX)) {
deserialize_ulong_long_io(geoms, &sf.index, geompos_in); deserialize_ulong_long_io(geoms, &sf.index, geompos_in);
} }
serial.hpp
+5 -5
View File
@@ -19,11 +19,11 @@ void serialize_ulong_long(FILE *out, unsigned long long n, std::atomic<long long
void serialize_byte(FILE *out, signed char n, std::atomic<long long> *fpos, const char *fname); void serialize_byte(FILE *out, signed char n, std::atomic<long long> *fpos, const char *fname);
void serialize_uint(FILE *out, unsigned n, std::atomic<long long> *fpos, const char *fname); void serialize_uint(FILE *out, unsigned n, std::atomic<long long> *fpos, const char *fname);
void serialize_int(std::string &out, int n, std::atomic<long long> *fpos); void serialize_int(std::string &out, int n);
void serialize_long_long(std::string &out, long long n, std::atomic<long long> *fpos); void serialize_long_long(std::string &out, long long n);
void serialize_ulong_long(std::string &out, unsigned long long n, std::atomic<long long> *fpos); void serialize_ulong_long(std::string &out, unsigned long long n);
void serialize_byte(std::string &out, signed char n, std::atomic<long long> *fpos); void serialize_byte(std::string &out, signed char n);
void serialize_uint(std::string &out, unsigned n, std::atomic<long long> *fpos); void serialize_uint(std::string &out, unsigned n);
void deserialize_int(char **f, int *n); void deserialize_int(char **f, int *n);
void deserialize_long_long(char **f, long long *n); void deserialize_long_long(char **f, long long *n);