mirror of
https://github.com/felt/tippecanoe.git
synced 2026-11-22 16:31:18 +01:00
Compare commits
43
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
40075c843a | ||
|
|
40a1022cc3 | ||
|
|
373227e422 | ||
|
|
d271cf233d | ||
|
|
1b176e55c3 | ||
|
|
8713b3a003 | ||
|
|
893f257f8f | ||
|
|
f5366eb767 | ||
|
|
4caa1d2758 | ||
|
|
1b1597f44b | ||
|
|
26d93a1a8d | ||
|
|
34926cf1dc | ||
|
|
c5f18671d8 | ||
|
|
b89cc9f425 | ||
|
|
69ebf3e988 | ||
|
|
5f5ee62a98 | ||
|
|
fb7d87145c | ||
|
|
8b0490ba35 | ||
|
|
a7bc8856fe | ||
|
|
98f6be7ebd | ||
|
|
12e3af018b | ||
|
|
5806be8a5b | ||
|
|
a409792fb5 | ||
|
|
da81ad3fed | ||
|
|
a035e4338a | ||
|
|
c7393d8e2e | ||
|
|
484dc27c5e | ||
|
|
21c946dd49 | ||
|
|
4c52aeacf1 | ||
|
|
f1bffd9de0 | ||
|
|
3addc57bfc | ||
|
|
fade093279 | ||
|
|
01aa8358f0 | ||
|
|
1ef0f260f1 | ||
|
|
2695db5ebf | ||
|
|
966fdb63d4 | ||
|
|
da2b5bd9fb | ||
|
|
42e2434cca | ||
|
|
69bf5b324b | ||
|
|
f59f8588eb | ||
|
|
fc8007a2ab | ||
|
|
eef29084ec | ||
|
|
d05f8f8e4c |
@@ -1,10 +1,3 @@
|
||||
## 2.23.0
|
||||
|
||||
* Remove the concept of "separate metadata." Features now always directly reference their keys and values rather than going through a second level of indirection.
|
||||
* Limit the size of the string pool to 1/5 the size of memory, to prevent thrashing during feature ingestion.
|
||||
* Avoid using writeable memory maps. Instead, explicitly copy data in and out of memory.
|
||||
* Compress streams of features in the temporary files, to reduce disk usage and I/O latency
|
||||
|
||||
## 2.22.0
|
||||
|
||||
* Speed up feature dropping by removing unnecessary search for other small features
|
||||
|
||||
@@ -47,7 +47,7 @@ C = $(wildcard *.c) $(wildcard *.cpp)
|
||||
INCLUDES = -I/usr/local/include -I.
|
||||
LIBS = -L/usr/local/lib
|
||||
|
||||
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o
|
||||
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o
|
||||
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
|
||||
|
||||
tippecanoe-enumerate: enumerate.o
|
||||
|
||||
-269
@@ -1,269 +0,0 @@
|
||||
#ifdef __APPLE__
|
||||
#define _DARWIN_UNLIMITED_STREAMS
|
||||
#endif
|
||||
|
||||
#include "compression.hpp"
|
||||
#include "errors.hpp"
|
||||
#include "protozero/varint.hpp"
|
||||
#include "serial.hpp"
|
||||
|
||||
void decompressor::begin() {
|
||||
within = true;
|
||||
|
||||
zs.zalloc = NULL;
|
||||
zs.zfree = NULL;
|
||||
zs.opaque = NULL;
|
||||
zs.msg = (char *) "";
|
||||
|
||||
int d = inflateInit(&zs);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "initialize decompression: %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
int decompressor::fread(void *p, size_t size, size_t nmemb, std::atomic<long long> *geompos) {
|
||||
zs.next_out = (Bytef *) p;
|
||||
zs.avail_out = size * nmemb;
|
||||
|
||||
while (zs.avail_out > 0) {
|
||||
if (zs.avail_in == 0) {
|
||||
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
|
||||
if (n == 0) {
|
||||
if (within) {
|
||||
fprintf(stderr, "Reached EOF while decompressing\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = n;
|
||||
}
|
||||
|
||||
size_t avail_before = zs.avail_in;
|
||||
|
||||
if (within) {
|
||||
int d = inflate(&zs, Z_NO_FLUSH);
|
||||
*geompos += avail_before - zs.avail_in;
|
||||
|
||||
if (d == Z_OK) {
|
||||
// it made some progress
|
||||
} else if (d == Z_STREAM_END) {
|
||||
// it may have made some progress and now we are done
|
||||
within = false;
|
||||
break;
|
||||
} else {
|
||||
fprintf(stderr, "decompression error %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
} else {
|
||||
size_t n = std::min(zs.avail_in, zs.avail_out);
|
||||
memcpy(zs.next_out, zs.next_in, n);
|
||||
*geompos += n;
|
||||
|
||||
zs.avail_out -= n;
|
||||
zs.avail_in -= n;
|
||||
zs.next_out += n;
|
||||
zs.next_in += n;
|
||||
}
|
||||
}
|
||||
|
||||
return (size * nmemb - zs.avail_out) / size;
|
||||
}
|
||||
|
||||
void decompressor::end(std::atomic<long long> *geompos) {
|
||||
// "within" means that we haven't received end-of-stream yet,
|
||||
// so consume more compressed data until we get there.
|
||||
// This can be necessary if the caller knows that it is at
|
||||
// the end of the feature stream (because it got a 0-length
|
||||
// feature) but the decompressor doesn't know yet.
|
||||
|
||||
if (within) {
|
||||
while (true) {
|
||||
if (zs.avail_in == 0) {
|
||||
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = n;
|
||||
}
|
||||
|
||||
zs.avail_out = 0;
|
||||
|
||||
size_t avail_before = zs.avail_in;
|
||||
int d = inflate(&zs, Z_NO_FLUSH);
|
||||
*geompos += avail_before - zs.avail_in;
|
||||
|
||||
if (d == Z_STREAM_END) {
|
||||
break;
|
||||
}
|
||||
if (d == Z_OK) {
|
||||
continue;
|
||||
}
|
||||
|
||||
fprintf(stderr, "decompression: got %d, not Z_STREAM_END\n", d);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
within = false;
|
||||
}
|
||||
|
||||
int d = inflateEnd(&zs);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "end decompression: %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
int decompressor::deserialize_ulong_long(unsigned long long *zigzag, std::atomic<long long> *geompos) {
|
||||
*zigzag = 0;
|
||||
int shift = 0;
|
||||
|
||||
while (1) {
|
||||
char c;
|
||||
if (fread(&c, sizeof(char), 1, geompos) != 1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ((c & 0x80) == 0) {
|
||||
*zigzag |= ((unsigned long long) c) << shift;
|
||||
shift += 7;
|
||||
break;
|
||||
} else {
|
||||
*zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
|
||||
shift += 7;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int decompressor::deserialize_long_long(long long *n, std::atomic<long long> *geompos) {
|
||||
unsigned long long zigzag = 0;
|
||||
int ret = deserialize_ulong_long(&zigzag, geompos);
|
||||
*n = protozero::decode_zigzag64(zigzag);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int decompressor::deserialize_int(int *n, std::atomic<long long> *geompos) {
|
||||
long long ll = 0;
|
||||
int ret = deserialize_long_long(&ll, geompos);
|
||||
*n = ll;
|
||||
return ret;
|
||||
}
|
||||
|
||||
int decompressor::deserialize_uint(unsigned *n, std::atomic<long long> *geompos) {
|
||||
unsigned long long v;
|
||||
deserialize_ulong_long(&v, geompos);
|
||||
*n = v;
|
||||
return 1;
|
||||
}
|
||||
|
||||
void compressor::begin() {
|
||||
zs.zalloc = NULL;
|
||||
zs.zfree = NULL;
|
||||
zs.opaque = NULL;
|
||||
zs.msg = (char *) "";
|
||||
|
||||
int d = deflateInit(&zs, Z_DEFAULT_COMPRESSION);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "initialize compression: %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
void compressor::compressor::end(std::atomic<long long> *fpos, const char *fname) {
|
||||
std::string buf;
|
||||
buf.resize(5000);
|
||||
|
||||
if (zs.avail_in != 0) {
|
||||
fprintf(stderr, "compression end called with data available\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = 0;
|
||||
|
||||
while (true) {
|
||||
zs.next_out = (Bytef *) buf.c_str();
|
||||
zs.avail_out = buf.size();
|
||||
|
||||
int d = deflate(&zs, Z_FINISH);
|
||||
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
|
||||
|
||||
if (d == Z_OK || d == Z_BUF_ERROR) {
|
||||
// it can take several calls to flush out all the buffered data
|
||||
continue;
|
||||
}
|
||||
|
||||
if (d != Z_STREAM_END) {
|
||||
fprintf(stderr, "%s: finish compression: %d %s\n", fname, d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
zs.next_out = (Bytef *) buf.c_str();
|
||||
zs.avail_out = buf.size();
|
||||
|
||||
int d = deflateEnd(&zs);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "%s: end compression: %d %s\n", fname, d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
|
||||
}
|
||||
|
||||
int compressor::fclose() {
|
||||
return ::fclose(fp);
|
||||
}
|
||||
|
||||
void compressor::fwrite_check(const char *p, size_t size, size_t nmemb, std::atomic<long long> *fpos, const char *fname) {
|
||||
std::string buf;
|
||||
buf.resize(size * nmemb * 2 + 200);
|
||||
|
||||
zs.next_in = (Bytef *) p;
|
||||
zs.avail_in = size * nmemb;
|
||||
|
||||
while (zs.avail_in > 0) {
|
||||
zs.next_out = (Bytef *) buf.c_str();
|
||||
zs.avail_out = buf.size();
|
||||
|
||||
int d = deflate(&zs, Z_NO_FLUSH);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "%s: deflate: %d %s\n", fname, d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
|
||||
}
|
||||
}
|
||||
|
||||
void compressor::serialize_ulong_long(unsigned long long val, std::atomic<long long> *fpos, const char *fname) {
|
||||
while (1) {
|
||||
unsigned char b = val & 0x7F;
|
||||
if ((val >> 7) != 0) {
|
||||
b |= 0x80;
|
||||
fwrite_check((const char *) &b, 1, 1, fpos, fname);
|
||||
val >>= 7;
|
||||
} else {
|
||||
fwrite_check((const char *) &b, 1, 1, fpos, fname);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void compressor::serialize_long_long(long long val, std::atomic<long long> *fpos, const char *fname) {
|
||||
unsigned long long zigzag = protozero::encode_zigzag64(val);
|
||||
|
||||
serialize_ulong_long(zigzag, fpos, fname);
|
||||
}
|
||||
|
||||
void compressor::serialize_int(int val, std::atomic<long long> *fpos, const char *fname) {
|
||||
serialize_long_long(val, fpos, fname);
|
||||
}
|
||||
|
||||
void compressor::serialize_uint(unsigned val, std::atomic<long long> *fpos, const char *fname) {
|
||||
serialize_ulong_long(val, fpos, fname);
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
#ifdef __APPLE__
|
||||
#define _DARWIN_UNLIMITED_STREAMS
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string>
|
||||
#include <atomic>
|
||||
#include <zlib.h>
|
||||
|
||||
struct decompressor {
|
||||
FILE *fp = NULL;
|
||||
z_stream zs;
|
||||
std::string buf;
|
||||
|
||||
// from begin() to receiving end-of-stream
|
||||
bool within = false;
|
||||
|
||||
decompressor(FILE *f) {
|
||||
fp = f;
|
||||
buf.resize(5000);
|
||||
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = 0;
|
||||
}
|
||||
|
||||
decompressor() {
|
||||
}
|
||||
|
||||
void begin();
|
||||
int fread(void *p, size_t size, size_t nmemb, std::atomic<long long> *geompos);
|
||||
void end(std::atomic<long long> *geompos);
|
||||
int deserialize_ulong_long(unsigned long long *zigzag, std::atomic<long long> *geompos);
|
||||
int deserialize_long_long(long long *n, std::atomic<long long> *geompos);
|
||||
int deserialize_int(int *n, std::atomic<long long> *geompos);
|
||||
int deserialize_uint(unsigned *n, std::atomic<long long> *geompos);
|
||||
};
|
||||
|
||||
struct compressor {
|
||||
FILE *fp = NULL;
|
||||
z_stream zs;
|
||||
|
||||
compressor(FILE *f) {
|
||||
fp = f;
|
||||
}
|
||||
|
||||
compressor() {
|
||||
}
|
||||
|
||||
void begin();
|
||||
void end(std::atomic<long long> *fpos, const char *fname);
|
||||
int fclose();
|
||||
void fwrite_check(const char *p, size_t size, size_t nmemb, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_ulong_long(unsigned long long val, std::atomic<long long> *fpos, const char *fname);
|
||||
|
||||
void serialize_long_long(long long val, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_int(int val, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_uint(unsigned val, std::atomic<long long> *fpos, const char *fname);
|
||||
};
|
||||
+4
-9
@@ -1538,8 +1538,7 @@ struct sorty {
|
||||
|
||||
struct sorty_sorter {
|
||||
int kind;
|
||||
sorty_sorter(int k)
|
||||
: kind(k){};
|
||||
sorty_sorter(int k) : kind(k) {};
|
||||
|
||||
bool operator()(const sorty &a, const sorty &b) const {
|
||||
long long xa, ya, xb, yb;
|
||||
@@ -1550,13 +1549,13 @@ struct sorty_sorter {
|
||||
|
||||
xb = b.x;
|
||||
yb = b.y;
|
||||
} else if (kind == 1) { // X first
|
||||
} else if (kind == 1) { // X first
|
||||
xa = a.y;
|
||||
ya = a.x;
|
||||
|
||||
xb = b.y;
|
||||
yb = b.x;
|
||||
} else if (kind == 2) { // diagonal
|
||||
} else if (kind == 2) { // diagonal
|
||||
xa = a.x + a.y;
|
||||
ya = a.x - a.y;
|
||||
|
||||
@@ -1773,11 +1772,7 @@ drawvec polygon_to_anchor(const drawvec &geom) {
|
||||
if (goodness <= 0) {
|
||||
double lon, lat;
|
||||
tile2lonlat(d.x, d.y, 32, &lon, &lat);
|
||||
|
||||
static std::atomic<long long> warned(0);
|
||||
if (warned++ < 10) {
|
||||
fprintf(stderr, "could not find good label point: %s %f,%f\n", kind, lat, lon);
|
||||
}
|
||||
fprintf(stderr, "could not find label point: %s %f,%f\n", kind, lat, lon);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -75,6 +75,8 @@ void memfile_full(struct memfile *file) {
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
fprintf(stderr, "Tippecanoe string pool is full (%zu); switching to file\n", file->off);
|
||||
|
||||
file->fp = fdopen(file->fd, "wb");
|
||||
if (file->fp == NULL) {
|
||||
fprintf(stderr, "fdopen memfile: %s\n", strerror(errno));
|
||||
|
||||
+58
-1
@@ -26,7 +26,7 @@
|
||||
|
||||
// Offset coordinates to keep them positive
|
||||
#define COORD_OFFSET (4LL << 32)
|
||||
#define SHIFT_RIGHT(a) ((long long) std::round((double) (a) / (1LL << geometry_scale)))
|
||||
#define SHIFT_RIGHT(a) ((long long) std::round((double)(a) / (1LL << geometry_scale)))
|
||||
#define SHIFT_LEFT(a) ((((a) + (COORD_OFFSET >> geometry_scale)) << geometry_scale) - COORD_OFFSET)
|
||||
|
||||
// write to file
|
||||
@@ -163,6 +163,63 @@ void deserialize_byte(char **f, signed char *n) {
|
||||
*f += sizeof(signed char);
|
||||
}
|
||||
|
||||
// read from file
|
||||
|
||||
int deserialize_long_long_io(FILE *f, long long *n, std::atomic<long long> *geompos) {
|
||||
unsigned long long zigzag = 0;
|
||||
int ret = deserialize_ulong_long_io(f, &zigzag, geompos);
|
||||
*n = protozero::decode_zigzag64(zigzag);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int deserialize_ulong_long_io(FILE *f, unsigned long long *zigzag, std::atomic<long long> *geompos) {
|
||||
*zigzag = 0;
|
||||
int shift = 0;
|
||||
|
||||
while (1) {
|
||||
int c = getc(f);
|
||||
if (c == EOF) {
|
||||
return 0;
|
||||
}
|
||||
(*geompos)++;
|
||||
|
||||
if ((c & 0x80) == 0) {
|
||||
*zigzag |= ((unsigned long long) c) << shift;
|
||||
shift += 7;
|
||||
break;
|
||||
} else {
|
||||
*zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
|
||||
shift += 7;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int deserialize_int_io(FILE *f, int *n, std::atomic<long long> *geompos) {
|
||||
long long ll = 0;
|
||||
int ret = deserialize_long_long_io(f, &ll, geompos);
|
||||
*n = ll;
|
||||
return ret;
|
||||
}
|
||||
|
||||
int deserialize_uint_io(FILE *f, unsigned *n, std::atomic<long long> *geompos) {
|
||||
unsigned long long v;
|
||||
deserialize_ulong_long_io(f, &v, geompos);
|
||||
*n = v;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int deserialize_byte_io(FILE *f, signed char *n, std::atomic<long long> *geompos) {
|
||||
int c = getc(f);
|
||||
if (c == EOF) {
|
||||
return 0;
|
||||
}
|
||||
*n = c;
|
||||
(*geompos)++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void write_geometry(drawvec const &dv, std::string &out, long long wx, long long wy) {
|
||||
for (size_t i = 0; i < dv.size(); i++) {
|
||||
if (dv[i].op == VT_MOVETO || dv[i].op == VT_LINETO) {
|
||||
|
||||
@@ -31,6 +31,12 @@ void deserialize_ulong_long(char **f, unsigned long long *n);
|
||||
void deserialize_uint(char **f, unsigned *n);
|
||||
void deserialize_byte(char **f, signed char *n);
|
||||
|
||||
int deserialize_int_io(FILE *f, int *n, std::atomic<long long> *geompos);
|
||||
int deserialize_long_long_io(FILE *f, long long *n, std::atomic<long long> *geompos);
|
||||
int deserialize_ulong_long_io(FILE *f, unsigned long long *n, std::atomic<long long> *geompos);
|
||||
int deserialize_uint_io(FILE *f, unsigned *n, std::atomic<long long> *geompos);
|
||||
int deserialize_byte_io(FILE *f, signed char *n, std::atomic<long long> *geompos);
|
||||
|
||||
struct serial_val {
|
||||
int type = 0;
|
||||
std::string s = "";
|
||||
|
||||
File diff suppressed because one or more lines are too long
Binary file not shown.
@@ -41,7 +41,6 @@
|
||||
#include "milo/dtoa_milo.h"
|
||||
#include "evaluator.hpp"
|
||||
#include "errors.hpp"
|
||||
#include "compression.hpp"
|
||||
#include "protozero/varint.hpp"
|
||||
|
||||
extern "C" {
|
||||
@@ -50,11 +49,285 @@ extern "C" {
|
||||
|
||||
#include "plugin.hpp"
|
||||
|
||||
struct decompressor {
|
||||
FILE *fp = NULL;
|
||||
z_stream zs;
|
||||
std::string buf;
|
||||
bool within = false;
|
||||
|
||||
decompressor(FILE *f) {
|
||||
fp = f;
|
||||
buf.resize(5000);
|
||||
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = 0;
|
||||
}
|
||||
|
||||
decompressor() { }
|
||||
|
||||
void begin() {
|
||||
within = true;
|
||||
|
||||
zs.zalloc = NULL;
|
||||
zs.zfree = NULL;
|
||||
zs.opaque = NULL;
|
||||
zs.msg = (char *) "";
|
||||
|
||||
int d = inflateInit(&zs);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "initialize decompression: %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
int fread(void *p, size_t size, size_t nmemb, std::atomic<long long> *geompos) {
|
||||
zs.next_out = (Bytef *) p;
|
||||
zs.avail_out = size * nmemb;
|
||||
|
||||
while (zs.avail_out > 0) {
|
||||
if (zs.avail_in == 0) {
|
||||
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
|
||||
if (n == 0) {
|
||||
if (within) {
|
||||
fprintf(stderr, "Reached EOF while decompressing\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = n;
|
||||
}
|
||||
|
||||
size_t avail_before = zs.avail_in;
|
||||
|
||||
if (within) {
|
||||
int d = inflate(&zs, Z_NO_FLUSH);
|
||||
*geompos += avail_before - zs.avail_in;
|
||||
|
||||
if (d == Z_OK) {
|
||||
// it made some progress
|
||||
} else if (d == Z_STREAM_END) {
|
||||
// it may have made some progress and now we are done
|
||||
within = false;
|
||||
break;
|
||||
} else {
|
||||
fprintf(stderr, "decompression error %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
} else {
|
||||
size_t n = std::min(zs.avail_in, zs.avail_out);
|
||||
memcpy(zs.next_out, zs.next_in, n);
|
||||
*geompos += n;
|
||||
|
||||
zs.avail_out -= n;
|
||||
zs.avail_in -= n;
|
||||
zs.next_out += n;
|
||||
zs.next_in += n;
|
||||
}
|
||||
}
|
||||
|
||||
return (size * nmemb - zs.avail_out) / size;
|
||||
}
|
||||
|
||||
void end(std::atomic<long long> *) {
|
||||
if (zs.avail_in == 0) {
|
||||
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = n;
|
||||
}
|
||||
|
||||
if (within) {
|
||||
zs.avail_out = 0;
|
||||
int d = inflate(&zs, Z_NO_FLUSH);
|
||||
if (d != Z_STREAM_END) {
|
||||
fprintf(stderr, "decompression: got %d, not Z_STREAM_END\n", d);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
within = false;
|
||||
|
||||
int d = inflateEnd(&zs);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "end decompression: %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
int deserialize_ulong_long(unsigned long long *zigzag, std::atomic<long long> *geompos) {
|
||||
*zigzag = 0;
|
||||
int shift = 0;
|
||||
|
||||
while (1) {
|
||||
char c;
|
||||
if (fread(&c, sizeof(char), 1, geompos) != 1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ((c & 0x80) == 0) {
|
||||
*zigzag |= ((unsigned long long) c) << shift;
|
||||
shift += 7;
|
||||
break;
|
||||
} else {
|
||||
*zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
|
||||
shift += 7;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int deserialize_long_long(long long *n, std::atomic<long long> *geompos) {
|
||||
unsigned long long zigzag = 0;
|
||||
int ret = deserialize_ulong_long(&zigzag, geompos);
|
||||
*n = protozero::decode_zigzag64(zigzag);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int deserialize_int(int *n, std::atomic<long long> *geompos) {
|
||||
long long ll = 0;
|
||||
int ret = deserialize_long_long(&ll, geompos);
|
||||
*n = ll;
|
||||
return ret;
|
||||
}
|
||||
|
||||
int deserialize_uint(unsigned *n, std::atomic<long long> *geompos) {
|
||||
unsigned long long v;
|
||||
deserialize_ulong_long(&v, geompos);
|
||||
*n = v;
|
||||
return 1;
|
||||
}
|
||||
};
|
||||
|
||||
struct compressor {
|
||||
FILE *fp = NULL;
|
||||
z_stream zs;
|
||||
|
||||
compressor(FILE *f) {
|
||||
fp = f;
|
||||
}
|
||||
|
||||
compressor() { }
|
||||
|
||||
void begin() {
|
||||
zs.zalloc = NULL;
|
||||
zs.zfree = NULL;
|
||||
zs.opaque = NULL;
|
||||
zs.msg = (char *) "";
|
||||
|
||||
int d = deflateInit(&zs, Z_DEFAULT_COMPRESSION);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "initialize compression: %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
void end(std::atomic<long long> *fpos, const char *fname) {
|
||||
std::string buf;
|
||||
buf.resize(5000);
|
||||
|
||||
if (zs.avail_in != 0) {
|
||||
fprintf(stderr, "compression end called with data available\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = 0;
|
||||
|
||||
while (true) {
|
||||
zs.next_out = (Bytef *) buf.c_str();
|
||||
zs.avail_out = buf.size();
|
||||
|
||||
int d = deflate(&zs, Z_FINISH);
|
||||
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
|
||||
|
||||
if (d == Z_OK || d == Z_BUF_ERROR) {
|
||||
// it can take several calls to flush out all the buffered data
|
||||
continue;
|
||||
}
|
||||
|
||||
if (d != Z_STREAM_END) {
|
||||
fprintf(stderr, "%s: finish compression: %d %s\n", fname, d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
zs.next_out = (Bytef *) buf.c_str();
|
||||
zs.avail_out = buf.size();
|
||||
|
||||
int d = deflateEnd(&zs);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "%s: end compression: %d %s\n", fname, d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
|
||||
|
||||
// ::fwrite_check("aaaaaaaaa", 1, 9, fp, fpos, fname);
|
||||
}
|
||||
|
||||
int fclose() {
|
||||
return ::fclose(fp);
|
||||
}
|
||||
|
||||
void fwrite_check(const char *p, size_t size, size_t nmemb, std::atomic<long long> *fpos, const char *fname) {
|
||||
std::string buf;
|
||||
buf.resize(size * nmemb * 2 + 200);
|
||||
|
||||
zs.next_in = (Bytef *) p;
|
||||
zs.avail_in = size * nmemb;
|
||||
|
||||
while (zs.avail_in > 0) {
|
||||
zs.next_out = (Bytef *) buf.c_str();
|
||||
zs.avail_out = buf.size();
|
||||
|
||||
int d = deflate(&zs, Z_NO_FLUSH);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "%s: deflate: %d %s\n", fname, d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
|
||||
}
|
||||
}
|
||||
|
||||
void serialize_ulong_long(unsigned long long val, std::atomic<long long> *fpos, const char *fname) {
|
||||
while (1) {
|
||||
unsigned char b = val & 0x7F;
|
||||
if ((val >> 7) != 0) {
|
||||
b |= 0x80;
|
||||
fwrite_check((const char *) &b, 1, 1, fpos, fname);
|
||||
val >>= 7;
|
||||
} else {
|
||||
fwrite_check((const char *) &b, 1, 1, fpos, fname);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void serialize_long_long(long long val, std::atomic<long long> *fpos, const char *fname) {
|
||||
unsigned long long zigzag = protozero::encode_zigzag64(val);
|
||||
|
||||
serialize_ulong_long(zigzag, fpos, fname);
|
||||
}
|
||||
|
||||
void serialize_int(int val, std::atomic<long long> *fpos, const char *fname) {
|
||||
serialize_long_long(val, fpos, fname);
|
||||
}
|
||||
|
||||
void serialize_uint(unsigned val, std::atomic<long long> *fpos, const char *fname) {
|
||||
serialize_ulong_long(val, fpos, fname);
|
||||
}
|
||||
};
|
||||
|
||||
#define CMD_BITS 3
|
||||
|
||||
// Offset coordinates to keep them positive
|
||||
#define COORD_OFFSET (4LL << 32)
|
||||
#define SHIFT_RIGHT(a) ((long long) std::round((double) (a) / (1LL << geometry_scale)))
|
||||
#define SHIFT_RIGHT(a) ((long long) std::round((double)(a) / (1LL << geometry_scale)))
|
||||
|
||||
#define XSTRINGIFY(s) STRINGIFY(s)
|
||||
#define STRINGIFY(s) #s
|
||||
@@ -1443,8 +1716,12 @@ serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos
|
||||
long long len;
|
||||
|
||||
if (geoms->deserialize_long_long(&len, geompos_in) == 0) {
|
||||
fprintf(stderr, "Unexpected physical EOF in feature stream\n");
|
||||
exit(EXIT_READ);
|
||||
if (compressed) {
|
||||
geoms->end(geompos_in);
|
||||
}
|
||||
|
||||
sf.t = -2;
|
||||
return sf;
|
||||
}
|
||||
if (len == 0) {
|
||||
if (compressed) {
|
||||
@@ -3015,7 +3292,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<
|
||||
}
|
||||
compressors[j] = compressor(fp);
|
||||
sub[j] = &compressors[j];
|
||||
unlink(geomname);
|
||||
// unlink(geomname);
|
||||
}
|
||||
|
||||
size_t useful_threads = 0;
|
||||
@@ -3026,6 +3303,11 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<
|
||||
useful_threads++;
|
||||
}
|
||||
}
|
||||
if (z == iz + 1) {
|
||||
if (!quiet) {
|
||||
fprintf(stderr, "(compressed geometry size %lld for zoom level %d)\n", todo, z);
|
||||
}
|
||||
}
|
||||
|
||||
size_t threads = CPUS;
|
||||
if (threads > TEMP_FILES / 4) {
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
#ifndef VERSION_HPP
|
||||
#define VERSION_HPP
|
||||
|
||||
#define VERSION "v2.23.0"
|
||||
#define VERSION "v2.22.0"
|
||||
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user