Compare commits

..
43 Commits
Author SHA1 Message Date
Erica Fischer 40075c843a Remove debug spew 2023-02-01 14:09:21 -08:00
Erica Fischer 40a1022cc3 Tests pass! 2023-02-01 14:01:46 -08:00
Erica Fischer 373227e422 Fix seeking within compressed feature streams 2023-02-01 13:58:12 -08:00
Erica Fischer d271cf233d Handle nonzero initial zooms 2023-02-01 13:32:58 -08:00
Erica Fischer 1b176e55c3 It made a tileset! 2023-02-01 13:11:06 -08:00
Erica Fischer 8713b3a003 One bug down 2023-02-01 12:35:36 -08:00
Erica Fischer 893f257f8f More attempts that don't work 2023-01-31 13:45:51 -08:00
Erica Fischer f5366eb767 Works until we get to concatenated tiles 2023-01-30 14:45:48 -08:00
Erica Fischer 4caa1d2758 Sort of works 2023-01-30 14:18:30 -08:00
Erica Fischer 1b1597f44b Closer, but still doesn't work 2023-01-30 13:53:07 -08:00
Erica Fischer 26d93a1a8d Work on alternating compressed and uncompressed in decompression 2023-01-30 13:32:24 -08:00
Erica Fischer 34926cf1dc Tile number is uncompressed, tile content is compressed 2023-01-30 13:21:32 -08:00
Erica Fischer c5f18671d8 Start on decompression 2023-01-27 15:19:32 -08:00
Erica Fischer b89cc9f425 Make enough deflate calls to flush out all the buffered data 2023-01-27 13:43:25 -08:00
Erica Fischer 69ebf3e988 Merge remote-tracking branch 'origin/main' into compression 2023-01-27 13:24:40 -08:00
Erica Fischer 5f5ee62a98 Merge branch 'compression' of github.com:felt/tippecanoe into compression 2023-01-27 13:23:53 -08:00
Erica Fischer fb7d87145c Remove unused code from the other day 2023-01-26 22:08:29 -08:00
Erica Fischer 8b0490ba35 Hook up more compression functions 2023-01-26 22:07:44 -08:00
Erica Fischer a7bc8856fe Starting toward compression again from a different point 2023-01-26 21:33:52 -08:00
Erica Fischer 98f6be7ebd Merge remote-tracking branch 'origin/main' into compression 2023-01-26 16:44:21 -08:00
Erica Fischer 12e3af018b Pull the writing of the serialized feature out to the callers 2023-01-25 14:41:22 -08:00
Erica Fischer 5806be8a5b Add compressed stream writer functions 2023-01-25 13:45:38 -08:00
Erica Fischer a409792fb5 Track file position within fwrite_check() 2023-01-25 13:09:34 -08:00
Erica Fischer da81ad3fed Start adding a stream compressor object 2023-01-25 11:19:04 -08:00
Erica Fischer a035e4338a Remove per-feature compression; move minzoom back into the object 2023-01-25 10:07:22 -08:00
Erica Fischer c7393d8e2e Don't need to store the length of the geometry 2023-01-24 13:26:12 -08:00
Erica Fischer 484dc27c5e Compress individual features in the temporary files 2023-01-24 13:24:36 -08:00
Erica Fischer 21c946dd49 Serialize features into memory rather than straight to disk 2023-01-24 12:19:58 -08:00
Erica Fischer 4c52aeacf1 Fix the pointer bug that was causing the crash 2023-01-24 10:17:25 -08:00
Erica Fischer f1bffd9de0 Crashy work in progress toward compression 2023-01-24 09:38:07 -08:00
Erica Fischer 3addc57bfc Add to-memory versions of the serialization functions 2023-01-23 15:47:38 -08:00
Erica Fischer fade093279 Merge remote-tracking branch 'origin/main' into no-separate-metadata 2023-01-23 10:53:27 -08:00
Erica Fischer 01aa8358f0 Switch string pool from memory to disk when memory is 10% full 2023-01-18 13:45:16 -08:00
Erica Fischer 1ef0f260f1 When the pool gets too big, switch to just the file, not memory 2023-01-17 11:04:26 -08:00
Erica Fischer 2695db5ebf Also use pread instead of mapping to read in the data to sort 2023-01-12 18:21:46 -08:00
Erica Fischer 966fdb63d4 Sort in allocated rather than mapped memory too 2023-01-12 18:19:43 -08:00
Erica Fischer da2b5bd9fb Move the string pool and search tree from mmap to allocated memory 2023-01-12 11:19:44 -08:00
Erica Fischer 42e2434cca Add missing #include 2023-01-03 11:13:59 -08:00
Erica Fischer 69bf5b324b Limit the size of the string pools and trees to fit in memory 2023-01-03 11:10:52 -08:00
Erica Fischer f59f8588eb Revert "Shave a few bytes off temporary files by using more unsigned integers"
This reverts commit eef29084ec.
2023-01-03 10:55:42 -08:00
Erica Fischer fc8007a2ab Flush stderr after logging progress 2022-12-30 12:38:37 -08:00
Erica Fischer eef29084ec Shave a few bytes off temporary files by using more unsigned integers 2022-12-30 11:46:03 -08:00
Erica Fischer d05f8f8e4c Remove the concept of "separate metadata"
This was an extra level of attribute indirection (features point
to metadata records which point to key and value strings) which was
intended to reduce the size of temporary storage for features with
large numbers of attributes that were also spread across large numbers
of tiles at maxzoom.

For other kinds of features, the extra indirection slowed things down
instead, and, especially when maxzoom guessing was being used, many more
features were having their metadata externalized than could actually
benefit from it.
2022-12-29 14:47:27 -08:00
12 changed files with 359 additions and 85237 deletions
-7
View File
@@ -1,10 +1,3 @@
## 2.23.0
* Remove the concept of "separate metadata." Features now always directly reference their keys and values rather than going through a second level of indirection.
* Limit the size of the string pool to 1/5 the size of memory, to prevent thrashing during feature ingestion.
* Avoid using writeable memory maps. Instead, explicitly copy data in and out of memory.
* Compress streams of features in the temporary files, to reduce disk usage and I/O latency
## 2.22.0
* Speed up feature dropping by removing unnecessary search for other small features
+1 -1
View File
@@ -47,7 +47,7 @@ C = $(wildcard *.c) $(wildcard *.cpp)
INCLUDES = -I/usr/local/include -I.
LIBS = -L/usr/local/lib
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
tippecanoe-enumerate: enumerate.o
-269
View File
@@ -1,269 +0,0 @@
#ifdef __APPLE__
#define _DARWIN_UNLIMITED_STREAMS
#endif
#include "compression.hpp"
#include "errors.hpp"
#include "protozero/varint.hpp"
#include "serial.hpp"
void decompressor::begin() {
within = true;
zs.zalloc = NULL;
zs.zfree = NULL;
zs.opaque = NULL;
zs.msg = (char *) "";
int d = inflateInit(&zs);
if (d != Z_OK) {
fprintf(stderr, "initialize decompression: %d %s\n", d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
}
int decompressor::fread(void *p, size_t size, size_t nmemb, std::atomic<long long> *geompos) {
zs.next_out = (Bytef *) p;
zs.avail_out = size * nmemb;
while (zs.avail_out > 0) {
if (zs.avail_in == 0) {
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
if (n == 0) {
if (within) {
fprintf(stderr, "Reached EOF while decompressing\n");
exit(EXIT_IMPOSSIBLE);
} else {
break;
}
}
zs.next_in = (Bytef *) buf.c_str();
zs.avail_in = n;
}
size_t avail_before = zs.avail_in;
if (within) {
int d = inflate(&zs, Z_NO_FLUSH);
*geompos += avail_before - zs.avail_in;
if (d == Z_OK) {
// it made some progress
} else if (d == Z_STREAM_END) {
// it may have made some progress and now we are done
within = false;
break;
} else {
fprintf(stderr, "decompression error %d %s\n", d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
} else {
size_t n = std::min(zs.avail_in, zs.avail_out);
memcpy(zs.next_out, zs.next_in, n);
*geompos += n;
zs.avail_out -= n;
zs.avail_in -= n;
zs.next_out += n;
zs.next_in += n;
}
}
return (size * nmemb - zs.avail_out) / size;
}
void decompressor::end(std::atomic<long long> *geompos) {
// "within" means that we haven't received end-of-stream yet,
// so consume more compressed data until we get there.
// This can be necessary if the caller knows that it is at
// the end of the feature stream (because it got a 0-length
// feature) but the decompressor doesn't know yet.
if (within) {
while (true) {
if (zs.avail_in == 0) {
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
zs.next_in = (Bytef *) buf.c_str();
zs.avail_in = n;
}
zs.avail_out = 0;
size_t avail_before = zs.avail_in;
int d = inflate(&zs, Z_NO_FLUSH);
*geompos += avail_before - zs.avail_in;
if (d == Z_STREAM_END) {
break;
}
if (d == Z_OK) {
continue;
}
fprintf(stderr, "decompression: got %d, not Z_STREAM_END\n", d);
exit(EXIT_IMPOSSIBLE);
}
within = false;
}
int d = inflateEnd(&zs);
if (d != Z_OK) {
fprintf(stderr, "end decompression: %d %s\n", d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
}
int decompressor::deserialize_ulong_long(unsigned long long *zigzag, std::atomic<long long> *geompos) {
*zigzag = 0;
int shift = 0;
while (1) {
char c;
if (fread(&c, sizeof(char), 1, geompos) != 1) {
return 0;
}
if ((c & 0x80) == 0) {
*zigzag |= ((unsigned long long) c) << shift;
shift += 7;
break;
} else {
*zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
shift += 7;
}
}
return 1;
}
int decompressor::deserialize_long_long(long long *n, std::atomic<long long> *geompos) {
unsigned long long zigzag = 0;
int ret = deserialize_ulong_long(&zigzag, geompos);
*n = protozero::decode_zigzag64(zigzag);
return ret;
}
int decompressor::deserialize_int(int *n, std::atomic<long long> *geompos) {
long long ll = 0;
int ret = deserialize_long_long(&ll, geompos);
*n = ll;
return ret;
}
int decompressor::deserialize_uint(unsigned *n, std::atomic<long long> *geompos) {
unsigned long long v;
deserialize_ulong_long(&v, geompos);
*n = v;
return 1;
}
void compressor::begin() {
zs.zalloc = NULL;
zs.zfree = NULL;
zs.opaque = NULL;
zs.msg = (char *) "";
int d = deflateInit(&zs, Z_DEFAULT_COMPRESSION);
if (d != Z_OK) {
fprintf(stderr, "initialize compression: %d %s\n", d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
}
void compressor::compressor::end(std::atomic<long long> *fpos, const char *fname) {
std::string buf;
buf.resize(5000);
if (zs.avail_in != 0) {
fprintf(stderr, "compression end called with data available\n");
exit(EXIT_IMPOSSIBLE);
}
zs.next_in = (Bytef *) buf.c_str();
zs.avail_in = 0;
while (true) {
zs.next_out = (Bytef *) buf.c_str();
zs.avail_out = buf.size();
int d = deflate(&zs, Z_FINISH);
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
if (d == Z_OK || d == Z_BUF_ERROR) {
// it can take several calls to flush out all the buffered data
continue;
}
if (d != Z_STREAM_END) {
fprintf(stderr, "%s: finish compression: %d %s\n", fname, d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
break;
}
zs.next_out = (Bytef *) buf.c_str();
zs.avail_out = buf.size();
int d = deflateEnd(&zs);
if (d != Z_OK) {
fprintf(stderr, "%s: end compression: %d %s\n", fname, d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
}
int compressor::fclose() {
return ::fclose(fp);
}
void compressor::fwrite_check(const char *p, size_t size, size_t nmemb, std::atomic<long long> *fpos, const char *fname) {
std::string buf;
buf.resize(size * nmemb * 2 + 200);
zs.next_in = (Bytef *) p;
zs.avail_in = size * nmemb;
while (zs.avail_in > 0) {
zs.next_out = (Bytef *) buf.c_str();
zs.avail_out = buf.size();
int d = deflate(&zs, Z_NO_FLUSH);
if (d != Z_OK) {
fprintf(stderr, "%s: deflate: %d %s\n", fname, d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
}
}
void compressor::serialize_ulong_long(unsigned long long val, std::atomic<long long> *fpos, const char *fname) {
while (1) {
unsigned char b = val & 0x7F;
if ((val >> 7) != 0) {
b |= 0x80;
fwrite_check((const char *) &b, 1, 1, fpos, fname);
val >>= 7;
} else {
fwrite_check((const char *) &b, 1, 1, fpos, fname);
break;
}
}
}
void compressor::serialize_long_long(long long val, std::atomic<long long> *fpos, const char *fname) {
unsigned long long zigzag = protozero::encode_zigzag64(val);
serialize_ulong_long(zigzag, fpos, fname);
}
void compressor::serialize_int(int val, std::atomic<long long> *fpos, const char *fname) {
serialize_long_long(val, fpos, fname);
}
void compressor::serialize_uint(unsigned val, std::atomic<long long> *fpos, const char *fname) {
serialize_ulong_long(val, fpos, fname);
}
-58
View File
@@ -1,58 +0,0 @@
#ifdef __APPLE__
#define _DARWIN_UNLIMITED_STREAMS
#endif
#include <stdio.h>
#include <string>
#include <atomic>
#include <zlib.h>
struct decompressor {
FILE *fp = NULL;
z_stream zs;
std::string buf;
// from begin() to receiving end-of-stream
bool within = false;
decompressor(FILE *f) {
fp = f;
buf.resize(5000);
zs.next_in = (Bytef *) buf.c_str();
zs.avail_in = 0;
}
decompressor() {
}
void begin();
int fread(void *p, size_t size, size_t nmemb, std::atomic<long long> *geompos);
void end(std::atomic<long long> *geompos);
int deserialize_ulong_long(unsigned long long *zigzag, std::atomic<long long> *geompos);
int deserialize_long_long(long long *n, std::atomic<long long> *geompos);
int deserialize_int(int *n, std::atomic<long long> *geompos);
int deserialize_uint(unsigned *n, std::atomic<long long> *geompos);
};
struct compressor {
FILE *fp = NULL;
z_stream zs;
compressor(FILE *f) {
fp = f;
}
compressor() {
}
void begin();
void end(std::atomic<long long> *fpos, const char *fname);
int fclose();
void fwrite_check(const char *p, size_t size, size_t nmemb, std::atomic<long long> *fpos, const char *fname);
void serialize_ulong_long(unsigned long long val, std::atomic<long long> *fpos, const char *fname);
void serialize_long_long(long long val, std::atomic<long long> *fpos, const char *fname);
void serialize_int(int val, std::atomic<long long> *fpos, const char *fname);
void serialize_uint(unsigned val, std::atomic<long long> *fpos, const char *fname);
};
+4 -9
View File
@@ -1538,8 +1538,7 @@ struct sorty {
struct sorty_sorter {
int kind;
sorty_sorter(int k)
: kind(k){};
sorty_sorter(int k) : kind(k) {};
bool operator()(const sorty &a, const sorty &b) const {
long long xa, ya, xb, yb;
@@ -1550,13 +1549,13 @@ struct sorty_sorter {
xb = b.x;
yb = b.y;
} else if (kind == 1) { // X first
} else if (kind == 1) { // X first
xa = a.y;
ya = a.x;
xb = b.y;
yb = b.x;
} else if (kind == 2) { // diagonal
} else if (kind == 2) { // diagonal
xa = a.x + a.y;
ya = a.x - a.y;
@@ -1773,11 +1772,7 @@ drawvec polygon_to_anchor(const drawvec &geom) {
if (goodness <= 0) {
double lon, lat;
tile2lonlat(d.x, d.y, 32, &lon, &lat);
static std::atomic<long long> warned(0);
if (warned++ < 10) {
fprintf(stderr, "could not find good label point: %s %f,%f\n", kind, lat, lon);
}
fprintf(stderr, "could not find label point: %s %f,%f\n", kind, lat, lon);
}
}
+2
View File
@@ -75,6 +75,8 @@ void memfile_full(struct memfile *file) {
exit(EXIT_IMPOSSIBLE);
}
fprintf(stderr, "Tippecanoe string pool is full (%zu); switching to file\n", file->off);
file->fp = fdopen(file->fd, "wb");
if (file->fp == NULL) {
fprintf(stderr, "fdopen memfile: %s\n", strerror(errno));
+58 -1
View File
@@ -26,7 +26,7 @@
// Offset coordinates to keep them positive
#define COORD_OFFSET (4LL << 32)
#define SHIFT_RIGHT(a) ((long long) std::round((double) (a) / (1LL << geometry_scale)))
#define SHIFT_RIGHT(a) ((long long) std::round((double)(a) / (1LL << geometry_scale)))
#define SHIFT_LEFT(a) ((((a) + (COORD_OFFSET >> geometry_scale)) << geometry_scale) - COORD_OFFSET)
// write to file
@@ -163,6 +163,63 @@ void deserialize_byte(char **f, signed char *n) {
*f += sizeof(signed char);
}
// read from file
int deserialize_long_long_io(FILE *f, long long *n, std::atomic<long long> *geompos) {
unsigned long long zigzag = 0;
int ret = deserialize_ulong_long_io(f, &zigzag, geompos);
*n = protozero::decode_zigzag64(zigzag);
return ret;
}
int deserialize_ulong_long_io(FILE *f, unsigned long long *zigzag, std::atomic<long long> *geompos) {
*zigzag = 0;
int shift = 0;
while (1) {
int c = getc(f);
if (c == EOF) {
return 0;
}
(*geompos)++;
if ((c & 0x80) == 0) {
*zigzag |= ((unsigned long long) c) << shift;
shift += 7;
break;
} else {
*zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
shift += 7;
}
}
return 1;
}
int deserialize_int_io(FILE *f, int *n, std::atomic<long long> *geompos) {
long long ll = 0;
int ret = deserialize_long_long_io(f, &ll, geompos);
*n = ll;
return ret;
}
int deserialize_uint_io(FILE *f, unsigned *n, std::atomic<long long> *geompos) {
unsigned long long v;
deserialize_ulong_long_io(f, &v, geompos);
*n = v;
return 1;
}
int deserialize_byte_io(FILE *f, signed char *n, std::atomic<long long> *geompos) {
int c = getc(f);
if (c == EOF) {
return 0;
}
*n = c;
(*geompos)++;
return 1;
}
static void write_geometry(drawvec const &dv, std::string &out, long long wx, long long wy) {
for (size_t i = 0; i < dv.size(); i++) {
if (dv[i].op == VT_MOVETO || dv[i].op == VT_LINETO) {
+6
View File
@@ -31,6 +31,12 @@ void deserialize_ulong_long(char **f, unsigned long long *n);
void deserialize_uint(char **f, unsigned *n);
void deserialize_byte(char **f, signed char *n);
int deserialize_int_io(FILE *f, int *n, std::atomic<long long> *geompos);
int deserialize_long_long_io(FILE *f, long long *n, std::atomic<long long> *geompos);
int deserialize_ulong_long_io(FILE *f, unsigned long long *n, std::atomic<long long> *geompos);
int deserialize_uint_io(FILE *f, unsigned *n, std::atomic<long long> *geompos);
int deserialize_byte_io(FILE *f, signed char *n, std::atomic<long long> *geompos);
struct serial_val {
int type = 0;
std::string s = "";
File diff suppressed because one or more lines are too long
Binary file not shown.
+287 -5
View File
@@ -41,7 +41,6 @@
#include "milo/dtoa_milo.h"
#include "evaluator.hpp"
#include "errors.hpp"
#include "compression.hpp"
#include "protozero/varint.hpp"
extern "C" {
@@ -50,11 +49,285 @@ extern "C" {
#include "plugin.hpp"
struct decompressor {
FILE *fp = NULL;
z_stream zs;
std::string buf;
bool within = false;
decompressor(FILE *f) {
fp = f;
buf.resize(5000);
zs.next_in = (Bytef *) buf.c_str();
zs.avail_in = 0;
}
decompressor() { }
void begin() {
within = true;
zs.zalloc = NULL;
zs.zfree = NULL;
zs.opaque = NULL;
zs.msg = (char *) "";
int d = inflateInit(&zs);
if (d != Z_OK) {
fprintf(stderr, "initialize decompression: %d %s\n", d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
}
int fread(void *p, size_t size, size_t nmemb, std::atomic<long long> *geompos) {
zs.next_out = (Bytef *) p;
zs.avail_out = size * nmemb;
while (zs.avail_out > 0) {
if (zs.avail_in == 0) {
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
if (n == 0) {
if (within) {
fprintf(stderr, "Reached EOF while decompressing\n");
exit(EXIT_IMPOSSIBLE);
} else {
break;
}
}
zs.next_in = (Bytef *) buf.c_str();
zs.avail_in = n;
}
size_t avail_before = zs.avail_in;
if (within) {
int d = inflate(&zs, Z_NO_FLUSH);
*geompos += avail_before - zs.avail_in;
if (d == Z_OK) {
// it made some progress
} else if (d == Z_STREAM_END) {
// it may have made some progress and now we are done
within = false;
break;
} else {
fprintf(stderr, "decompression error %d %s\n", d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
} else {
size_t n = std::min(zs.avail_in, zs.avail_out);
memcpy(zs.next_out, zs.next_in, n);
*geompos += n;
zs.avail_out -= n;
zs.avail_in -= n;
zs.next_out += n;
zs.next_in += n;
}
}
return (size * nmemb - zs.avail_out) / size;
}
void end(std::atomic<long long> *) {
if (zs.avail_in == 0) {
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
zs.next_in = (Bytef *) buf.c_str();
zs.avail_in = n;
}
if (within) {
zs.avail_out = 0;
int d = inflate(&zs, Z_NO_FLUSH);
if (d != Z_STREAM_END) {
fprintf(stderr, "decompression: got %d, not Z_STREAM_END\n", d);
exit(EXIT_FAILURE);
}
}
within = false;
int d = inflateEnd(&zs);
if (d != Z_OK) {
fprintf(stderr, "end decompression: %d %s\n", d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
}
int deserialize_ulong_long(unsigned long long *zigzag, std::atomic<long long> *geompos) {
*zigzag = 0;
int shift = 0;
while (1) {
char c;
if (fread(&c, sizeof(char), 1, geompos) != 1) {
return 0;
}
if ((c & 0x80) == 0) {
*zigzag |= ((unsigned long long) c) << shift;
shift += 7;
break;
} else {
*zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
shift += 7;
}
}
return 1;
}
int deserialize_long_long(long long *n, std::atomic<long long> *geompos) {
unsigned long long zigzag = 0;
int ret = deserialize_ulong_long(&zigzag, geompos);
*n = protozero::decode_zigzag64(zigzag);
return ret;
}
int deserialize_int(int *n, std::atomic<long long> *geompos) {
long long ll = 0;
int ret = deserialize_long_long(&ll, geompos);
*n = ll;
return ret;
}
int deserialize_uint(unsigned *n, std::atomic<long long> *geompos) {
unsigned long long v;
deserialize_ulong_long(&v, geompos);
*n = v;
return 1;
}
};
struct compressor {
FILE *fp = NULL;
z_stream zs;
compressor(FILE *f) {
fp = f;
}
compressor() { }
void begin() {
zs.zalloc = NULL;
zs.zfree = NULL;
zs.opaque = NULL;
zs.msg = (char *) "";
int d = deflateInit(&zs, Z_DEFAULT_COMPRESSION);
if (d != Z_OK) {
fprintf(stderr, "initialize compression: %d %s\n", d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
}
void end(std::atomic<long long> *fpos, const char *fname) {
std::string buf;
buf.resize(5000);
if (zs.avail_in != 0) {
fprintf(stderr, "compression end called with data available\n");
exit(EXIT_IMPOSSIBLE);
}
zs.next_in = (Bytef *) buf.c_str();
zs.avail_in = 0;
while (true) {
zs.next_out = (Bytef *) buf.c_str();
zs.avail_out = buf.size();
int d = deflate(&zs, Z_FINISH);
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
if (d == Z_OK || d == Z_BUF_ERROR) {
// it can take several calls to flush out all the buffered data
continue;
}
if (d != Z_STREAM_END) {
fprintf(stderr, "%s: finish compression: %d %s\n", fname, d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
break;
}
zs.next_out = (Bytef *) buf.c_str();
zs.avail_out = buf.size();
int d = deflateEnd(&zs);
if (d != Z_OK) {
fprintf(stderr, "%s: end compression: %d %s\n", fname, d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
// ::fwrite_check("aaaaaaaaa", 1, 9, fp, fpos, fname);
}
int fclose() {
return ::fclose(fp);
}
void fwrite_check(const char *p, size_t size, size_t nmemb, std::atomic<long long> *fpos, const char *fname) {
std::string buf;
buf.resize(size * nmemb * 2 + 200);
zs.next_in = (Bytef *) p;
zs.avail_in = size * nmemb;
while (zs.avail_in > 0) {
zs.next_out = (Bytef *) buf.c_str();
zs.avail_out = buf.size();
int d = deflate(&zs, Z_NO_FLUSH);
if (d != Z_OK) {
fprintf(stderr, "%s: deflate: %d %s\n", fname, d, zs.msg);
exit(EXIT_IMPOSSIBLE);
}
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
}
}
void serialize_ulong_long(unsigned long long val, std::atomic<long long> *fpos, const char *fname) {
while (1) {
unsigned char b = val & 0x7F;
if ((val >> 7) != 0) {
b |= 0x80;
fwrite_check((const char *) &b, 1, 1, fpos, fname);
val >>= 7;
} else {
fwrite_check((const char *) &b, 1, 1, fpos, fname);
break;
}
}
}
void serialize_long_long(long long val, std::atomic<long long> *fpos, const char *fname) {
unsigned long long zigzag = protozero::encode_zigzag64(val);
serialize_ulong_long(zigzag, fpos, fname);
}
void serialize_int(int val, std::atomic<long long> *fpos, const char *fname) {
serialize_long_long(val, fpos, fname);
}
void serialize_uint(unsigned val, std::atomic<long long> *fpos, const char *fname) {
serialize_ulong_long(val, fpos, fname);
}
};
#define CMD_BITS 3
// Offset coordinates to keep them positive
#define COORD_OFFSET (4LL << 32)
#define SHIFT_RIGHT(a) ((long long) std::round((double) (a) / (1LL << geometry_scale)))
#define SHIFT_RIGHT(a) ((long long) std::round((double)(a) / (1LL << geometry_scale)))
#define XSTRINGIFY(s) STRINGIFY(s)
#define STRINGIFY(s) #s
@@ -1443,8 +1716,12 @@ serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos
long long len;
if (geoms->deserialize_long_long(&len, geompos_in) == 0) {
fprintf(stderr, "Unexpected physical EOF in feature stream\n");
exit(EXIT_READ);
if (compressed) {
geoms->end(geompos_in);
}
sf.t = -2;
return sf;
}
if (len == 0) {
if (compressed) {
@@ -3015,7 +3292,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<
}
compressors[j] = compressor(fp);
sub[j] = &compressors[j];
unlink(geomname);
// unlink(geomname);
}
size_t useful_threads = 0;
@@ -3026,6 +3303,11 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<
useful_threads++;
}
}
if (z == iz + 1) {
if (!quiet) {
fprintf(stderr, "(compressed geometry size %lld for zoom level %d)\n", todo, z);
}
}
size_t threads = CPUS;
if (threads > TEMP_FILES / 4) {
+1 -1
View File
@@ -1,6 +1,6 @@
#ifndef VERSION_HPP
#define VERSION_HPP
#define VERSION "v2.23.0"
#define VERSION "v2.22.0"
#endif