Compress individual features in the temporary files

mvt.cpp

@@ -82,14 +82,14 @@ int decompress(std::string const &input, std::string &output) {
 }
 
 // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
-int compress(std::string const &input, std::string &output) {
+int compress(std::string const &input, std::string &output, bool gz) {
 	z_stream deflate_s;
 	deflate_s.zalloc = Z_NULL;
 	deflate_s.zfree = Z_NULL;
 	deflate_s.opaque = Z_NULL;
 	deflate_s.avail_in = 0;
 	deflate_s.next_in = Z_NULL;
-	deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
+	deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, gz ? 31 : 15, 8, Z_DEFAULT_STRATEGY);
 	deflate_s.next_in = (Bytef *) input.data();
 	deflate_s.avail_in = input.size();
 	size_t length = 0;

mvt.hpp

@@ -115,7 +115,7 @@ struct mvt_tile {
 
 bool is_compressed(std::string const &data);
 int decompress(std::string const &input, std::string &output);
-int compress(std::string const &input, std::string &output);
+int compress(std::string const &input, std::string &output, bool gz);
 int dezig(unsigned n);
 
 mvt_value stringified_to_mvt_value(int type, const char *s);

pmtiles_file.cpp

@@ -57,7 +57,7 @@ std::string compress_fn(const std::string &input, uint8_t compression) {
 	if (compression == pmtiles::COMPRESSION_NONE) {
 		output = input;
 	} else if (compression == pmtiles::COMPRESSION_GZIP) {
-		compress(input, output);
+		compress(input, output, true);
 	} else {
 		throw std::runtime_error("Unknown or unsupported compression.");
 	}
@@ -121,7 +121,7 @@ std::string metadata_to_pmtiles_json(metadata m) {
 	state.json_end_hash();
 	state.json_write_newline();
 	std::string compressed;
-	compress(buf, compressed);
+	compress(buf, compressed, true);
 	return compressed;
 }
 

serial.cpp

@@ -12,6 +12,7 @@
 #include "protozero/varint.hpp"
 #include "geometry.hpp"
 #include "mbtiles.hpp"
+#include "mvt.hpp"
 #include "tile.hpp"
 #include "serial.hpp"
 #include "options.hpp"
@@ -297,19 +298,35 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long
 		serialize_long_long(s, sf->values[i]);
 	}
 
-	serialize_byte(s, sf->feature_minzoom);
+	if (s.size() > 500) {  // smaller than this rarely comes out smaller than it started
+		std::string compressed;
+		compress(s, compressed, false);  // false for no gzip header: saves 12 bytes
 
-	serialize_ulong_long(geomfile, s.size(), geompos, fname);
-	fwrite_check(s.c_str(), sizeof(char), s.size(), geomfile, fname);
-	*geompos += s.size();
+		if (s.size() <= compressed.size()) {
+			serialize_long_long(geomfile, s.size(), geompos, fname);
+			fwrite_check(s.c_str(), sizeof(char), s.size(), geomfile, fname);
+			*geompos += s.size();
+		} else {
+			serialize_long_long(geomfile, -compressed.size(), geompos, fname);
+			fwrite_check(compressed.c_str(), sizeof(char), compressed.size(), geomfile, fname);
+			*geompos += compressed.size();
+		}
+	} else {
+		serialize_long_long(geomfile, s.size(), geompos, fname);
+		fwrite_check(s.c_str(), sizeof(char), s.size(), geomfile, fname);
+		*geompos += s.size();
+	}
+
+	// MAGIC: This knows that the feature minzoom is the last byte of the feature,
+	serialize_byte(geomfile, sf->feature_minzoom, geompos, fname);
 }
 
 serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
 	serial_feature sf;
 	std::string s;
-	unsigned long long len;
+	long long len;
 
-	if (deserialize_ulong_long_io(geoms, &len, geompos_in) == 0) {
+	if (deserialize_long_long_io(geoms, &len, geompos_in) == 0) {
 		sf.t = -2;
 		return sf;
 	}
@@ -318,14 +335,24 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_
 		return sf;
 	}
 
-	s.resize(len);
-	size_t n = fread((void *) s.c_str(), sizeof(char), len, geoms);
-	if (n != len) {
+	s.resize(std::abs(len));
+	size_t n = fread((void *) s.c_str(), sizeof(char), s.size(), geoms);
+	if (n != s.size()) {
 		fprintf(stderr, "Short read (%zu for %zu) from geometry\n", n, s.size());
 		exit(EXIT_READ);
 	}
 	*geompos_in += n;
 
+	if (len < 0) {
+		std::string d;
+		if (decompress(s, d) == 0) {
+			fprintf(stderr, "Internal decompression failure\n");
+			exit(EXIT_IMPOSSIBLE);
+		}
+		s = d;
+		len = s.size();
+	}
+
 	char *cp = (char *) s.c_str();
 
 	deserialize_byte(&cp, &sf.t);
@@ -387,13 +414,14 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_
 		sf.values.push_back(v);
 	}
 
-	deserialize_byte(&cp, &sf.feature_minzoom);
-
 	if (cp != s.c_str() + len) {
 		fprintf(stderr, "wrong length decoding feature: used %zd, len is %llu\n", cp - s.c_str(), len);
 		exit(EXIT_IMPOSSIBLE);
 	}
 
+	// MAGIC: This knows that the feature minzoom is the last byte of the feature.
+	deserialize_byte_io(geoms, &sf.feature_minzoom, geompos_in);
+
 	return sf;
 }
 

tile-join.cpp

@@ -544,7 +544,7 @@ void *join_worker(void *v) {
 			std::string compressed;
 
 			if (!pC) {
-				compress(pbf, compressed);
+				compress(pbf, compressed, true);
 			} else {
 				compressed = pbf;
 			}

tile.cpp

@@ -2659,7 +2659,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *stri
 			std::string pbf = tile.encode();
 
 			if (!prevent[P_TILE_COMPRESSION]) {
-				compress(pbf, compressed);
+				compress(pbf, compressed, true);
 			} else {
 				compressed = pbf;
 			}