diff --git a/mvt.cpp b/mvt.cpp index e8e5348f..b4365863 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -82,14 +82,14 @@ int decompress(std::string const &input, std::string &output) { } // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp -int compress(std::string const &input, std::string &output) { +int compress(std::string const &input, std::string &output, bool gz) { z_stream deflate_s; deflate_s.zalloc = Z_NULL; deflate_s.zfree = Z_NULL; deflate_s.opaque = Z_NULL; deflate_s.avail_in = 0; deflate_s.next_in = Z_NULL; - deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); + deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, gz ? 31 : 15, 8, Z_DEFAULT_STRATEGY); deflate_s.next_in = (Bytef *) input.data(); deflate_s.avail_in = input.size(); size_t length = 0; diff --git a/mvt.hpp b/mvt.hpp index 8afaddff..696e2535 100644 --- a/mvt.hpp +++ b/mvt.hpp @@ -115,7 +115,7 @@ struct mvt_tile { bool is_compressed(std::string const &data); int decompress(std::string const &input, std::string &output); -int compress(std::string const &input, std::string &output); +int compress(std::string const &input, std::string &output, bool gz); int dezig(unsigned n); mvt_value stringified_to_mvt_value(int type, const char *s); diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index 5de7d844..cf6f0759 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -57,7 +57,7 @@ std::string compress_fn(const std::string &input, uint8_t compression) { if (compression == pmtiles::COMPRESSION_NONE) { output = input; } else if (compression == pmtiles::COMPRESSION_GZIP) { - compress(input, output); + compress(input, output, true); } else { throw std::runtime_error("Unknown or unsupported compression."); } @@ -121,7 +121,7 @@ std::string metadata_to_pmtiles_json(metadata m) { state.json_end_hash(); state.json_write_newline(); std::string compressed; - compress(buf, compressed); + compress(buf, compressed, true); return compressed; } diff --git a/serial.cpp b/serial.cpp index a2436016..fd00e268 100644 --- a/serial.cpp +++ b/serial.cpp @@ -12,6 +12,7 @@ #include "protozero/varint.hpp" #include "geometry.hpp" #include "mbtiles.hpp" +#include "mvt.hpp" #include "tile.hpp" #include "serial.hpp" #include "options.hpp" @@ -297,19 +298,35 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomicvalues[i]); } - serialize_byte(s, sf->feature_minzoom); + if (s.size() > 500) { // smaller than this rarely comes out smaller than it started + std::string compressed; + compress(s, compressed, false); // false for no gzip header: saves 12 bytes - serialize_ulong_long(geomfile, s.size(), geompos, fname); - fwrite_check(s.c_str(), sizeof(char), s.size(), geomfile, fname); - *geompos += s.size(); + if (s.size() <= compressed.size()) { + serialize_long_long(geomfile, s.size(), geompos, fname); + fwrite_check(s.c_str(), sizeof(char), s.size(), geomfile, fname); + *geompos += s.size(); + } else { + serialize_long_long(geomfile, -compressed.size(), geompos, fname); + fwrite_check(compressed.c_str(), sizeof(char), compressed.size(), geomfile, fname); + *geompos += compressed.size(); + } + } else { + serialize_long_long(geomfile, s.size(), geompos, fname); + fwrite_check(s.c_str(), sizeof(char), s.size(), geomfile, fname); + *geompos += s.size(); + } + + // MAGIC: This knows that the feature minzoom is the last byte of the feature, + serialize_byte(geomfile, sf->feature_minzoom, geompos, fname); } serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) { serial_feature sf; std::string s; - unsigned long long len; + long long len; - if (deserialize_ulong_long_io(geoms, &len, geompos_in) == 0) { + if (deserialize_long_long_io(geoms, &len, geompos_in) == 0) { sf.t = -2; return sf; } @@ -318,14 +335,24 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_ return sf; } - s.resize(len); - size_t n = fread((void *) s.c_str(), sizeof(char), len, geoms); - if (n != len) { + s.resize(std::abs(len)); + size_t n = fread((void *) s.c_str(), sizeof(char), s.size(), geoms); + if (n != s.size()) { fprintf(stderr, "Short read (%zu for %zu) from geometry\n", n, s.size()); exit(EXIT_READ); } *geompos_in += n; + if (len < 0) { + std::string d; + if (decompress(s, d) == 0) { + fprintf(stderr, "Internal decompression failure\n"); + exit(EXIT_IMPOSSIBLE); + } + s = d; + len = s.size(); + } + char *cp = (char *) s.c_str(); deserialize_byte(&cp, &sf.t); @@ -387,13 +414,14 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_ sf.values.push_back(v); } - deserialize_byte(&cp, &sf.feature_minzoom); - if (cp != s.c_str() + len) { fprintf(stderr, "wrong length decoding feature: used %zd, len is %llu\n", cp - s.c_str(), len); exit(EXIT_IMPOSSIBLE); } + // MAGIC: This knows that the feature minzoom is the last byte of the feature. + deserialize_byte_io(geoms, &sf.feature_minzoom, geompos_in); + return sf; } diff --git a/tile-join.cpp b/tile-join.cpp index 6ea572b2..c988e4eb 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -544,7 +544,7 @@ void *join_worker(void *v) { std::string compressed; if (!pC) { - compress(pbf, compressed); + compress(pbf, compressed, true); } else { compressed = pbf; } diff --git a/tile.cpp b/tile.cpp index ca6559a9..df6a2342 100644 --- a/tile.cpp +++ b/tile.cpp @@ -2659,7 +2659,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *stri std::string pbf = tile.encode(); if (!prevent[P_TILE_COMPRESSION]) { - compress(pbf, compressed); + compress(pbf, compressed, true); } else { compressed = pbf; }