Merge remote-tracking branch 'origin/master' into earcut-polygon

.travis.yml

@@ -28,6 +28,8 @@ install:
        export CXXFLAGS="--coverage -g";
        export CFLAGS="--coverage -g";
        export LDFLAGS="--coverage";
+   elif [[ $(uname -s) == 'Linux' ]]; then
+       export CXX=clang++;
    fi;
  - make
 

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.9.16
+
+* Switch to protozero as the library for reading and writing protocol buffers
+
 ## 1.9.15
 
 * Add option not to clip features

Makefile

@@ -24,28 +24,25 @@ install: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join
 man/tippecanoe.1: README.md
 	md2man-roff README.md > man/tippecanoe.1
 
-vector_tile.pb.cc vector_tile.pb.h: vector_tile.proto
-	protoc --cpp_out=. vector_tile.proto
-
 PG=
 
 H = $(shell find . '(' -name '*.h' -o -name '*.hh' ')')
 C = $(shell find . '(' -name '*.c' -o -name '*.cc' ')')
 
-INCLUDES = -I/usr/local/include
+INCLUDES = -I/usr/local/include -I.
 LIBS = -L/usr/local/lib
 
-tippecanoe: geojson.o jsonpull.o vector_tile.pb.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o
+tippecanoe: geojson.o jsonpull.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o mvt.o
-	$(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lprotobuf-lite -lsqlite3 -lpthread
+	$(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 tippecanoe-enumerate: enumerate.o
 	$(CC) $(PG) $(LIBS) -O3 -g -Wall $(CFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3
 
-tippecanoe-decode: decode.o vector_tile.pb.o projection.o
+tippecanoe-decode: decode.o projection.o mvt.o
-	$(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lprotobuf-lite -lsqlite3
+	$(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
-tile-join: tile-join.o vector_tile.pb.o projection.o pool.o mbtiles.o
+tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o
-	$(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lprotobuf-lite -lsqlite3
+	$(CXX) $(PG) $(LIBS) -O3 -g -Wall $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
 libjsonpull.a: jsonpull.o
 	ar rc $@ $^

README.md

@@ -231,16 +231,12 @@ lower resolutions before failing if it still doesn't fit.
 Development
 -----------
 
-Requires protoc and sqlite3. Rebuilding the manpage
+Requires sqlite3 (should already be installed on MacOS). Rebuilding the manpage
 uses md2man (`gem install md2man`).
 
-MacOS:
-
-    brew install protobuf
-
 Linux:
 
-    sudo apt-get install libprotobuf-dev protobuf-compiler libsqlite3-dev
+    sudo apt-get install libsqlite3-dev
 
 Then build:
 

decode.cc

@@ -3,6 +3,8 @@
 #include <unistd.h>
 #include <sqlite3.h>
 #include <string>
+#include <vector>
+#include <map>
 #include <zlib.h>
 #include <math.h>
 #include <fcntl.h>
@@ -10,53 +12,13 @@
 #include <sys/mman.h>
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 #include <google/protobuf/io/coded_stream.h>
-#include "vector_tile.pb.h"
+#include "mvt.hh"
 #include "tile.h"
 
 extern "C" {
 #include "projection.h"
 }
 
-// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
-inline bool is_compressed(std::string const &data) {
-	return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B));
-}
-
-// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
-inline int decompress(std::string const &input, std::string &output) {
-	z_stream inflate_s;
-	inflate_s.zalloc = Z_NULL;
-	inflate_s.zfree = Z_NULL;
-	inflate_s.opaque = Z_NULL;
-	inflate_s.avail_in = 0;
-	inflate_s.next_in = Z_NULL;
-	if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) {
-		fprintf(stderr, "error: %s\n", inflate_s.msg);
-	}
-	inflate_s.next_in = (Bytef *) input.data();
-	inflate_s.avail_in = input.size();
-	size_t length = 0;
-	do {
-		output.resize(length + 2 * input.size());
-		inflate_s.avail_out = 2 * input.size();
-		inflate_s.next_out = (Bytef *) (output.data() + length);
-		int ret = inflate(&inflate_s, Z_FINISH);
-		if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
-			fprintf(stderr, "error: %s\n", inflate_s.msg);
-			return 0;
-		}
-
-		length += (2 * input.size() - inflate_s.avail_out);
-	} while (inflate_s.avail_out == 0);
-	inflateEnd(&inflate_s);
-	output.resize(length);
-	return 1;
-}
-
-int dezig(unsigned n) {
-	return (n >> 1) ^ (-(n & 1));
-}
-
 void printq(const char *s) {
 	putchar('"');
 	for (; *s; s++) {
@@ -84,23 +46,10 @@ struct draw {
 };
 
 void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
-	GOOGLE_PROTOBUF_VERIFY_VERSION;
 	int within = 0;
+	mvt_tile tile;
 
-	// https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp
+	if (!tile.decode(message)) {
-	mapnik::vector::tile tile;
-
-	if (is_compressed(message)) {
-		std::string uncompressed;
-		decompress(message, uncompressed);
-		google::protobuf::io::ArrayInputStream stream(uncompressed.c_str(), uncompressed.length());
-		google::protobuf::io::CodedInputStream codedstream(&stream);
-		codedstream.SetTotalBytesLimit(10 * 67108864, 5 * 67108864);
-		if (!tile.ParseFromCodedStream(&codedstream)) {
-			fprintf(stderr, "Couldn't decompress tile %d/%u/%u\n", z, x, y);
-			exit(EXIT_FAILURE);
-		}
-	} else if (!tile.ParseFromString(message)) {
 		fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y);
 		exit(EXIT_FAILURE);
 	}
@@ -113,9 +62,9 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
 
 	printf(", \"features\": [\n");
 
-	for (int l = 0; l < tile.layers_size(); l++) {
+	for (size_t l = 0; l < tile.layers.size(); l++) {
-		mapnik::vector::tile_layer layer = tile.layers(l);
+		mvt_layer &layer = tile.layers[l];
-		int extent = layer.extent();
+		int extent = layer.extent;
 
 		if (describe) {
 			if (l != 0) {
@@ -124,16 +73,15 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
 
 			printf("{ \"type\": \"FeatureCollection\"");
 			printf(", \"properties\": { \"layer\": ");
-			printq(layer.name().c_str());
+			printq(layer.name.c_str());
 			printf(" }");
 			printf(", \"features\": [\n");
 
 			within = 0;
 		}
 
-		for (int f = 0; f < layer.features_size(); f++) {
+		for (size_t f = 0; f < layer.features.size(); f++) {
-			mapnik::vector::tile_feature feat = layer.features(f);
+			mvt_feature &feat = layer.features[f];
-			int px = 0, py = 0;
 
 			if (within) {
 				printf(",\n");
@@ -143,46 +91,55 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
 			printf("{ \"type\": \"Feature\"");
 			printf(", \"properties\": { ");
 
-			for (int t = 0; t + 1 < feat.tags_size(); t += 2) {
+			for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) {
 				if (t != 0) {
 					printf(", ");
 				}
 
-				const char *key = layer.keys(feat.tags(t)).c_str();
+				if (feat.tags[t] >= layer.keys.size()) {
-				mapnik::vector::tile_value const &val = layer.values(feat.tags(t + 1));
+					fprintf(stderr, "Error: out of bounds feature key\n");
+					exit(EXIT_FAILURE);
+				}
+				if (feat.tags[t + 1] >= layer.values.size()) {
+					fprintf(stderr, "Error: out of bounds feature value\n");
+					exit(EXIT_FAILURE);
+				}
 
-				if (val.has_string_value()) {
+				const char *key = layer.keys[feat.tags[t]].c_str();
+				mvt_value const &val = layer.values[feat.tags[t + 1]];
+
+				if (val.type == mvt_string) {
 					printq(key);
 					printf(": ");
-					printq(val.string_value().c_str());
+					printq(val.string_value.c_str());
-				} else if (val.has_int_value()) {
+				} else if (val.type == mvt_int) {
 					printq(key);
-					printf(": %lld", (long long) val.int_value());
+					printf(": %lld", (long long) val.numeric_value.int_value);
-				} else if (val.has_double_value()) {
+				} else if (val.type == mvt_double) {
 					printq(key);
-					double v = val.double_value();
+					double v = val.numeric_value.double_value;
 					if (v == (long long) v) {
 						printf(": %lld", (long long) v);
 					} else {
 						printf(": %g", v);
 					}
-				} else if (val.has_float_value()) {
+				} else if (val.type == mvt_float) {
 					printq(key);
-					double v = val.float_value();
+					double v = val.numeric_value.float_value;
 					if (v == (long long) v) {
 						printf(": %lld", (long long) v);
 					} else {
 						printf(": %g", v);
 					}
-				} else if (val.has_sint_value()) {
+				} else if (val.type == mvt_sint) {
 					printq(key);
-					printf(": %lld", (long long) val.sint_value());
+					printf(": %lld", (long long) val.numeric_value.sint_value);
-				} else if (val.has_uint_value()) {
+				} else if (val.type == mvt_uint) {
 					printq(key);
-					printf(": %lld", (long long) val.uint_value());
+					printf(": %lld", (long long) val.numeric_value.uint_value);
-				} else if (val.has_bool_value()) {
+				} else if (val.type == mvt_bool) {
 					printq(key);
-					printf(": %s", val.bool_value() ? "true" : "false");
+					printf(": %s", val.numeric_value.bool_value ? "true" : "false");
 				}
 			}
 
@@ -190,32 +147,26 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
 
 			std::vector<draw> ops;
 
-			for (int g = 0; g < feat.geometry_size(); g++) {
+			for (size_t g = 0; g < feat.geometry.size(); g++) {
-				uint32_t geom = feat.geometry(g);
+				int op = feat.geometry[g].op;
-				uint32_t op = geom & 7;
+				long long px = feat.geometry[g].x;
-				uint32_t count = geom >> 3;
+				long long py = feat.geometry[g].y;
 
 				if (op == VT_MOVETO || op == VT_LINETO) {
-					for (size_t k = 0; k < count; k++) {
+					long long scale = 1LL << (32 - z);
-						px += dezig(feat.geometry(g + 1));
+					long long wx = scale * x + (scale / extent) * px;
-						py += dezig(feat.geometry(g + 2));
+					long long wy = scale * y + (scale / extent) * py;
-						g += 2;
 
-						long long scale = 1LL << (32 - z);
+					double lat, lon;
-						long long wx = scale * x + (scale / extent) * px;
+					tile2latlon(wx, wy, 32, &lat, &lon);
-						long long wy = scale * y + (scale / extent) * py;
 
-						double lat, lon;
+					ops.push_back(draw(op, lon, lat));
-						tile2latlon(wx, wy, 32, &lat, &lon);
-
-						ops.push_back(draw(op, lon, lat));
-					}
 				} else {
 					ops.push_back(draw(op, 0, 0));
 				}
 			}
 
-			if (feat.type() == VT_POINT) {
+			if (feat.type == VT_POINT) {
 				if (ops.size() == 1) {
 					printf("\"type\": \"Point\", \"coordinates\": [ %f, %f ]", ops[0].lon, ops[0].lat);
 				} else {
@@ -228,7 +179,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
 					}
 					printf(" ]");
 				}
-			} else if (feat.type() == VT_LINE) {
+			} else if (feat.type == VT_LINE) {
 				int movetos = 0;
 				for (size_t i = 0; i < ops.size(); i++) {
 					if (ops[i].op == VT_MOVETO) {
@@ -264,7 +215,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
 					}
 					printf(" ] ]");
 				}
-			} else if (feat.type() == VT_POLYGON) {
+			} else if (feat.type == VT_POLYGON) {
 				std::vector<std::vector<draw> > rings;
 				std::vector<double> areas;
 

man/tippecanoe.1

@@ -267,22 +267,14 @@ If a tile is larger than 500K, it will try encoding that tile at progressively
 lower resolutions before failing if it still doesn't fit.
 .SH Development
 .PP
-Requires protoc and sqlite3. Rebuilding the manpage
+Requires sqlite3 (should already be installed on MacOS). Rebuilding the manpage
 uses md2man (\fB\fCgem install md2man\fR).
 .PP
-MacOS:
-.PP
-.RS
-.nf
-brew install protobuf
-.fi
-.RE
-.PP
 Linux:
 .PP
 .RS
 .nf
-sudo apt\-get install libprotobuf\-dev protobuf\-compiler libsqlite3\-dev
+sudo apt\-get install libsqlite3\-dev
 .fi
 .RE
 .PP

mvt.cc

@@ -0,0 +1,409 @@
+#include <stdio.h>
+#include <string.h>
+#include <string>
+#include <vector>
+#include <map>
+#include <zlib.h>
+#include "mvt.hh"
+#include "protozero/pbf_reader.hpp"
+#include "protozero/pbf_writer.hpp"
+
+mvt_geometry::mvt_geometry(int op, long long x, long long y) {
+	this->op = op;
+	this->x = x;
+	this->y = y;
+}
+
+// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
+bool is_compressed(std::string const &data) {
+	return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B));
+}
+
+// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
+int decompress(std::string const &input, std::string &output) {
+	z_stream inflate_s;
+	inflate_s.zalloc = Z_NULL;
+	inflate_s.zfree = Z_NULL;
+	inflate_s.opaque = Z_NULL;
+	inflate_s.avail_in = 0;
+	inflate_s.next_in = Z_NULL;
+	if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) {
+		fprintf(stderr, "error: %s\n", inflate_s.msg);
+	}
+	inflate_s.next_in = (Bytef *) input.data();
+	inflate_s.avail_in = input.size();
+	size_t length = 0;
+	do {
+		output.resize(length + 2 * input.size());
+		inflate_s.avail_out = 2 * input.size();
+		inflate_s.next_out = (Bytef *) (output.data() + length);
+		int ret = inflate(&inflate_s, Z_FINISH);
+		if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
+			fprintf(stderr, "error: %s\n", inflate_s.msg);
+			return 0;
+		}
+
+		length += (2 * input.size() - inflate_s.avail_out);
+	} while (inflate_s.avail_out == 0);
+	inflateEnd(&inflate_s);
+	output.resize(length);
+	return 1;
+}
+
+// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
+int compress(std::string const &input, std::string &output) {
+	z_stream deflate_s;
+	deflate_s.zalloc = Z_NULL;
+	deflate_s.zfree = Z_NULL;
+	deflate_s.opaque = Z_NULL;
+	deflate_s.avail_in = 0;
+	deflate_s.next_in = Z_NULL;
+	deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
+	deflate_s.next_in = (Bytef *) input.data();
+	deflate_s.avail_in = input.size();
+	size_t length = 0;
+	do {
+		size_t increase = input.size() / 2 + 1024;
+		output.resize(length + increase);
+		deflate_s.avail_out = increase;
+		deflate_s.next_out = (Bytef *) (output.data() + length);
+		int ret = deflate(&deflate_s, Z_FINISH);
+		if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
+			return -1;
+		}
+		length += (increase - deflate_s.avail_out);
+	} while (deflate_s.avail_out == 0);
+	deflateEnd(&deflate_s);
+	output.resize(length);
+	return 0;
+}
+
+int dezig(unsigned n) {
+	return (n >> 1) ^ (-(n & 1));
+}
+
+bool mvt_tile::decode(std::string &message) {
+	layers.clear();
+	std::string src;
+
+	if (is_compressed(message)) {
+		std::string uncompressed;
+		decompress(message, uncompressed);
+		src = uncompressed;
+	} else {
+		src = message;
+	}
+
+	protozero::pbf_reader reader(src);
+
+	while (reader.next()) {
+		switch (reader.tag()) {
+		case 3: /* layer */
+		{
+			protozero::pbf_reader layer_reader(reader.get_message());
+			mvt_layer layer;
+
+			while (layer_reader.next()) {
+				switch (layer_reader.tag()) {
+				case 1: /* name */
+					layer.name = layer_reader.get_string();
+					break;
+
+				case 3: /* key */
+					layer.keys.push_back(layer_reader.get_string());
+					break;
+
+				case 4: /* value */
+				{
+					protozero::pbf_reader value_reader(layer_reader.get_message());
+					mvt_value value;
+
+					while (value_reader.next()) {
+						switch (value_reader.tag()) {
+						case 1: /* string */
+							value.type = mvt_string;
+							value.string_value = value_reader.get_string();
+							break;
+
+						case 2: /* float */
+							value.type = mvt_float;
+							value.numeric_value.float_value = value_reader.get_float();
+							break;
+
+						case 3: /* double */
+							value.type = mvt_double;
+							value.numeric_value.double_value = value_reader.get_double();
+							break;
+
+						case 4: /* int */
+							value.type = mvt_int;
+							value.numeric_value.int_value = value_reader.get_int64();
+							break;
+
+						case 5: /* uint */
+							value.type = mvt_uint;
+							value.numeric_value.uint_value = value_reader.get_uint64();
+							break;
+
+						case 6: /* sint */
+							value.type = mvt_sint;
+							value.numeric_value.sint_value = value_reader.get_sint64();
+							break;
+
+						case 7: /* bool */
+							value.type = mvt_bool;
+							value.numeric_value.bool_value = value_reader.get_bool();
+							break;
+
+						default:
+							value_reader.skip();
+							break;
+						}
+					}
+
+					layer.values.push_back(value);
+					break;
+				}
+
+				case 5: /* extent */
+					layer.extent = layer_reader.get_uint32();
+					break;
+
+				case 2: /* feature */
+				{
+					protozero::pbf_reader feature_reader(layer_reader.get_message());
+					mvt_feature feature;
+					std::vector<uint32_t> geoms;
+
+					while (feature_reader.next()) {
+						switch (feature_reader.tag()) {
+						case 2: /* tag */
+						{
+							auto pi = feature_reader.get_packed_uint32();
+							for (auto it = pi.first; it != pi.second; ++it) {
+								feature.tags.push_back(*it);
+							}
+							break;
+						}
+
+						case 3: /* feature type */
+							feature.type = feature_reader.get_enum();
+							break;
+
+						case 4: /* geometry */
+						{
+							auto pi = feature_reader.get_packed_uint32();
+							for (auto it = pi.first; it != pi.second; ++it) {
+								geoms.push_back(*it);
+							}
+							break;
+						}
+
+						default:
+							feature_reader.skip();
+							break;
+						}
+					}
+
+					long long px = 0, py = 0;
+					for (size_t g = 0; g < geoms.size(); g++) {
+						uint32_t geom = geoms[g];
+						uint32_t op = geom & 7;
+						uint32_t count = geom >> 3;
+
+						if (op == mvt_moveto || op == mvt_lineto) {
+							for (size_t k = 0; k < count && g + 2 < geoms.size(); k++) {
+								px += dezig(geoms[g + 1]);
+								py += dezig(geoms[g + 2]);
+								g += 2;
+
+								feature.geometry.push_back(mvt_geometry(op, px, py));
+							}
+						} else {
+							feature.geometry.push_back(mvt_geometry(op, 0, 0));
+						}
+					}
+
+					layer.features.push_back(feature);
+					break;
+				}
+
+				default:
+					layer_reader.skip();
+					break;
+				}
+			}
+
+			for (size_t i = 0; i < layer.keys.size(); i++) {
+				layer.key_map.insert(std::pair<std::string, size_t>(layer.keys[i], i));
+			}
+			for (size_t i = 0; i < layer.values.size(); i++) {
+				layer.value_map.insert(std::pair<mvt_value, size_t>(layer.values[i], i));
+			}
+
+			layers.push_back(layer);
+			break;
+		}
+
+		default:
+			reader.skip();
+			break;
+		}
+	}
+
+	return true;
+}
+
+std::string mvt_tile::encode() {
+	std::string data;
+
+	protozero::pbf_writer writer(data);
+
+	for (size_t i = 0; i < layers.size(); i++) {
+		std::string layer_string;
+		protozero::pbf_writer layer_writer(layer_string);
+
+		layer_writer.add_uint32(15, layers[i].version); /* version */
+		layer_writer.add_string(1, layers[i].name);     /* name */
+		layer_writer.add_uint32(5, layers[i].extent);   /* extent */
+
+		for (size_t j = 0; j < layers[i].keys.size(); j++) {
+			layer_writer.add_string(3, layers[i].keys[j]); /* key */
+		}
+
+		for (size_t v = 0; v < layers[i].values.size(); v++) {
+			std::string value_string;
+			protozero::pbf_writer value_writer(value_string);
+			mvt_value &pbv = layers[i].values[v];
+
+			if (pbv.type == mvt_string) {
+				value_writer.add_string(1, pbv.string_value);
+			} else if (pbv.type == mvt_float) {
+				value_writer.add_float(2, pbv.numeric_value.float_value);
+			} else if (pbv.type == mvt_double) {
+				value_writer.add_double(3, pbv.numeric_value.double_value);
+			} else if (pbv.type == mvt_int) {
+				value_writer.add_int64(4, pbv.numeric_value.int_value);
+			} else if (pbv.type == mvt_uint) {
+				value_writer.add_uint64(5, pbv.numeric_value.uint_value);
+			} else if (pbv.type == mvt_sint) {
+				value_writer.add_sint64(6, pbv.numeric_value.sint_value);
+			} else if (pbv.type == mvt_bool) {
+				value_writer.add_bool(7, pbv.numeric_value.bool_value);
+			}
+
+			layer_writer.add_message(4, value_string);
+		}
+
+		for (size_t f = 0; f < layers[i].features.size(); f++) {
+			std::string feature_string;
+			protozero::pbf_writer feature_writer(feature_string);
+
+			feature_writer.add_enum(3, layers[i].features[f].type);
+			feature_writer.add_packed_uint32(2, std::begin(layers[i].features[f].tags), std::end(layers[i].features[f].tags));
+
+			std::vector<uint32_t> geometry;
+
+			int px = 0, py = 0;
+			int cmd_idx = -1;
+			int cmd = -1;
+			int length = 0;
+
+			std::vector<mvt_geometry> &geom = layers[i].features[f].geometry;
+
+			for (size_t g = 0; g < geom.size(); g++) {
+				int op = geom[g].op;
+
+				if (op != cmd) {
+					if (cmd_idx >= 0) {
+						geometry[cmd_idx] = (length << 3) | (cmd & ((1 << 3) - 1));
+					}
+
+					cmd = op;
+					length = 0;
+					cmd_idx = geometry.size();
+					geometry.push_back(0);
+				}
+
+				if (op == mvt_moveto || op == mvt_lineto) {
+					long long wwx = geom[g].x;
+					long long wwy = geom[g].y;
+
+					int dx = wwx - px;
+					int dy = wwy - py;
+
+					geometry.push_back((dx << 1) ^ (dx >> 31));
+					geometry.push_back((dy << 1) ^ (dy >> 31));
+
+					px = wwx;
+					py = wwy;
+					length++;
+				} else if (op == mvt_closepath) {
+					length++;
+				} else {
+					fprintf(stderr, "\nInternal error: corrupted geometry\n");
+					exit(EXIT_FAILURE);
+				}
+			}
+
+			if (cmd_idx >= 0) {
+				geometry[cmd_idx] = (length << 3) | (cmd & ((1 << 3) - 1));
+			}
+
+			feature_writer.add_packed_uint32(4, std::begin(geometry), std::end(geometry));
+			layer_writer.add_message(2, feature_string);
+		}
+
+		writer.add_message(3, layer_string);
+	}
+
+	std::string compressed;
+	compress(data, compressed);
+
+	return compressed;
+}
+
+bool mvt_value::operator<(const mvt_value &o) const {
+	if (type < o.type) {
+		return true;
+	}
+	if (type == o.type) {
+		if ((type == mvt_string && string_value < o.string_value) ||
+		    (type == mvt_float && numeric_value.float_value < o.numeric_value.float_value) ||
+		    (type == mvt_double && numeric_value.double_value < o.numeric_value.double_value) ||
+		    (type == mvt_int && numeric_value.int_value < o.numeric_value.int_value) ||
+		    (type == mvt_uint && numeric_value.uint_value < o.numeric_value.uint_value) ||
+		    (type == mvt_sint && numeric_value.sint_value < o.numeric_value.sint_value) ||
+		    (type == mvt_bool && numeric_value.bool_value < o.numeric_value.bool_value)) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+void mvt_layer::tag(mvt_feature &feature, std::string key, mvt_value value) {
+	size_t ko, vo;
+
+	std::map<std::string, size_t>::iterator ki = key_map.find(key);
+	std::map<mvt_value, size_t>::iterator vi = value_map.find(value);
+
+	if (ki == key_map.end()) {
+		ko = keys.size();
+		keys.push_back(key);
+		key_map.insert(std::pair<std::string, size_t>(key, ko));
+	} else {
+		ko = ki->second;
+	}
+
+	if (vi == value_map.end()) {
+		vo = values.size();
+		values.push_back(value);
+		value_map.insert(std::pair<mvt_value, size_t>(value, vo));
+	} else {
+		vo = vi->second;
+	}
+
+	feature.tags.push_back(ko);
+	feature.tags.push_back(vo);
+}

mvt.hh

@@ -0,0 +1,81 @@
+struct mvt_value;
+struct mvt_layer;
+
+enum mvt_operation {
+	mvt_moveto = 1,
+	mvt_lineto = 2,
+	mvt_closepath = 7
+};
+
+struct mvt_geometry {
+	int /* mvt_operation */ op;
+	long long x;
+	long long y;
+
+	mvt_geometry(int op, long long x, long long y);
+};
+
+enum mvt_geometry_type {
+	mvt_point = 1,
+	mvt_linestring = 2,
+	mvt_polygon = 3
+};
+
+struct mvt_feature {
+	std::vector<unsigned> tags;
+	int /* mvt_geometry_type */ type;
+	std::vector<mvt_geometry> geometry;
+};
+
+enum mvt_value_type {
+	mvt_string,
+	mvt_float,
+	mvt_double,
+	mvt_int,
+	mvt_uint,
+	mvt_sint,
+	mvt_bool
+};
+
+struct mvt_value {
+	mvt_value_type type;
+	std::string string_value;
+	union {
+		float float_value;
+		double double_value;
+		long long int_value;
+		unsigned long long uint_value;
+		long long sint_value;
+		bool bool_value;
+	} numeric_value;
+
+	bool operator<(const mvt_value &o) const;
+};
+
+struct mvt_layer {
+	int version;
+	std::string name;
+	std::vector<mvt_feature> features;
+	std::vector<std::string> keys;
+	std::vector<mvt_value> values;
+	int extent;
+
+	// Add a key-value pair to a feature, using this layer's constant pool
+	void tag(mvt_feature &feature, std::string key, mvt_value value);
+
+	// For tracking the key-value constants already used in this layer
+	std::map<std::string, size_t> key_map;
+	std::map<mvt_value, size_t> value_map;
+};
+
+struct mvt_tile {
+	std::vector<mvt_layer> layers;
+
+	std::string encode();
+	bool decode(std::string &message);
+};
+
+bool is_compressed(std::string const &data);
+int decompress(std::string const &input, std::string &output);
+int compress(std::string const &input, std::string &output);
+int dezig(unsigned n);

protozero/byteswap.hpp

@@ -0,0 +1,71 @@
+#ifndef PROTOZERO_BYTESWAP_HPP
+#define PROTOZERO_BYTESWAP_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file byteswap.hpp
+ *
+ * @brief Contains functions to swap bytes in values (for different endianness).
+ */
+
+#include <cstdint>
+#include <cassert>
+
+#include <protozero/config.hpp>
+
+namespace protozero {
+
+/**
+ * Swap N byte value between endianness formats. This template function must
+ * be specialized to actually work.
+ */
+template <int N>
+inline void byteswap(const char* /*data*/, char* /*result*/) {
+    static_assert(N == 1, "Can only swap 4 or 8 byte values");
+}
+
+/**
+ * Swap 4 byte value (int32_t, uint32_t, float) between endianness formats.
+ */
+template <>
+inline void byteswap<4>(const char* data, char* result) {
+#ifdef PROTOZERO_USE_BUILTIN_BSWAP
+    *reinterpret_cast<uint32_t*>(result) = __builtin_bswap32(*reinterpret_cast<const uint32_t*>(data));
+#else
+    result[3] = data[0];
+    result[2] = data[1];
+    result[1] = data[2];
+    result[0] = data[3];
+#endif
+}
+
+/**
+ * Swap 8 byte value (int64_t, uint64_t, double) between endianness formats.
+ */
+template <>
+inline void byteswap<8>(const char* data, char* result) {
+#ifdef PROTOZERO_USE_BUILTIN_BSWAP
+    *reinterpret_cast<uint64_t*>(result) = __builtin_bswap64(*reinterpret_cast<const uint64_t*>(data));
+#else
+    result[7] = data[0];
+    result[6] = data[1];
+    result[5] = data[2];
+    result[4] = data[3];
+    result[3] = data[4];
+    result[2] = data[5];
+    result[1] = data[6];
+    result[0] = data[7];
+#endif
+}
+
+} // end namespace protozero
+
+#endif // PROTOZERO_BYTESWAP_HPP

protozero/config.hpp

@@ -0,0 +1,59 @@
+#ifndef PROTOZERO_CONFIG_HPP
+#define PROTOZERO_CONFIG_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+#include <cassert>
+
+/**
+ * @file config.hpp
+ *
+ * @brief Contains macro checks for different configurations.
+ */
+
+#define PROTOZERO_LITTLE_ENDIAN 1234
+#define PROTOZERO_BIG_ENDIAN    4321
+
+// Find out which byte order the machine has.
+#if defined(__BYTE_ORDER)
+# if (__BYTE_ORDER == __LITTLE_ENDIAN)
+#  define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN
+# endif
+# if (__BYTE_ORDER == __BIG_ENDIAN)
+#  define PROTOZERO_BYTE_ORDER PROTOZERO_BIG_ENDIAN
+# endif
+#else
+// This probably isn't a very good default, but might do until we figure
+// out something better.
+# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN
+#endif
+
+// On some ARM machines and depending on compiler settings access to unaligned
+// floating point values will result in a SIGBUS. Do not use the bare pointers
+// in this case.
+#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN
+# if !defined(__arm__) && !defined(_M_ARM)
+#  ifndef PROTOZERO_DO_NOT_USE_BARE_POINTER
+#   define PROTOZERO_USE_BARE_POINTER_FOR_PACKED_FIXED
+#  endif
+# endif
+#endif
+
+// Check whether __builtin_bswap is available
+#if defined(__GNUC__) || defined(__clang__)
+# define PROTOZERO_USE_BUILTIN_BSWAP
+#endif
+
+// Wrapper for assert() used for testing
+#ifndef protozero_assert
+# define protozero_assert(x) assert(x)
+#endif
+
+#endif // PROTOZERO_CONFIG_HPP

protozero/exception.hpp

@@ -0,0 +1,68 @@
+#ifndef PROTOZERO_EXCEPTION_HPP
+#define PROTOZERO_EXCEPTION_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file exception.hpp
+ *
+ * @brief Contains the exceptions used in the protozero library.
+ */
+
+#include <exception>
+
+/**
+ * @brief All parts of the protozero header-only library are in this namespace.
+ */
+namespace protozero {
+
+/**
+ * All exceptions explicitly thrown by the functions of the protozero library
+ * derive from this exception.
+ */
+struct exception : std::exception {
+    /// Returns the explanatory string.
+    const char *what() const noexcept override { return "pbf exception"; }
+};
+
+/**
+ * This exception is thrown when parsing a varint thats larger than allowed.
+ * This should never happen unless the data is corrupted.
+ */
+struct varint_too_long_exception : exception {
+    /// Returns the explanatory string.
+    const char *what() const noexcept override { return "varint too long exception"; }
+};
+
+/**
+ * This exception is thrown when the wire type of a pdf field is unknown.
+ * This should never happen unless the data is corrupted.
+ */
+struct unknown_pbf_wire_type_exception : exception {
+    /// Returns the explanatory string.
+    const char *what() const noexcept override { return "unknown pbf field type exception"; }
+};
+
+/**
+ * This exception is thrown when we are trying to read a field and there
+ * are not enough bytes left in the buffer to read it. Almost all functions
+ * of the pbf_reader class can throw this exception.
+ *
+ * This should never happen unless the data is corrupted or you have
+ * initialized the pbf_reader object with incomplete data.
+ */
+struct end_of_buffer_exception : exception {
+    /// Returns the explanatory string.
+    const char *what() const noexcept override { return "end of buffer exception"; }
+};
+
+} // end namespace protozero
+
+#endif // PROTOZERO_EXCEPTION_HPP

protozero/pbf_builder.hpp

@@ -0,0 +1,139 @@
+#ifndef PROTOZERO_PBF_BUILDER_HPP
+#define PROTOZERO_PBF_BUILDER_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file pbf_builder.hpp
+ *
+ * @brief Contains the pbf_builder template class.
+ */
+
+#include <type_traits>
+
+#include <protozero/types.hpp>
+#include <protozero/pbf_writer.hpp>
+
+namespace protozero {
+
+/**
+ * The pbf_builder is used to write PBF formatted messages into a buffer. It
+ * is based on the pbf_writer class and has all the same methods. The
+ * difference is that while the pbf_writer class takes an integer tag,
+ * this template class takes a tag of the template type T. The idea is that
+ * T will be an enumeration value and this helps reduce the possibility of
+ * programming errors.
+ *
+ * Almost all methods in this class can throw an std::bad_alloc exception if
+ * the std::string used as a buffer wants to resize.
+ *
+ * Read the tutorial to understand how this class is used.
+ */
+template <typename T>
+class pbf_builder : public pbf_writer {
+
+    static_assert(std::is_same<pbf_tag_type, typename std::underlying_type<T>::type>::value,
+                  "T must be enum with underlying type protozero::pbf_tag_type");
+
+public:
+
+    using enum_type = T;
+
+    pbf_builder(std::string& data) noexcept :
+        pbf_writer(data) {
+    }
+
+    template <typename P>
+    pbf_builder(pbf_writer& parent_writer, P tag) noexcept :
+        pbf_writer(parent_writer, pbf_tag_type(tag)) {
+    }
+
+/// @cond INTERNAL
+#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \
+    inline void add_##name(T tag, type value) { \
+        pbf_writer::add_##name(pbf_tag_type(tag), value); \
+    }
+
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float)
+    PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double)
+
+#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR
+/// @endcond
+
+    inline void add_bytes(T tag, const char* value, std::size_t size) {
+        pbf_writer::add_bytes(pbf_tag_type(tag), value, size);
+    }
+
+    inline void add_bytes(T tag, const std::string& value) {
+        pbf_writer::add_bytes(pbf_tag_type(tag), value);
+    }
+
+    inline void add_string(T tag, const char* value, std::size_t size) {
+        pbf_writer::add_string(pbf_tag_type(tag), value, size);
+    }
+
+    inline void add_string(T tag, const std::string& value) {
+        pbf_writer::add_string(pbf_tag_type(tag), value);
+    }
+
+    inline void add_string(T tag, const char* value) {
+        pbf_writer::add_string(pbf_tag_type(tag), value);
+    }
+
+    inline void add_message(T tag, const char* value, std::size_t size) {
+        pbf_writer::add_message(pbf_tag_type(tag), value, size);
+    }
+
+    inline void add_message(T tag, const std::string& value) {
+        pbf_writer::add_message(pbf_tag_type(tag), value);
+    }
+
+/// @cond INTERNAL
+#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \
+    template <typename InputIterator> \
+    inline void add_packed_##name(T tag, InputIterator first, InputIterator last) { \
+        pbf_writer::add_packed_##name(pbf_tag_type(tag), first, last); \
+    }
+
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(bool)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(enum)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(int32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(int64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(float)
+    PROTOZERO_WRITER_WRAP_ADD_PACKED(double)
+
+#undef PROTOZERO_WRITER_WRAP_ADD_PACKED
+/// @endcond
+
+};
+
+} // end namespace protozero
+
+#endif // PROTOZERO_PBF_BUILDER_HPP

protozero/pbf_message.hpp

@@ -0,0 +1,94 @@
+#ifndef PROTOZERO_PBF_MESSAGE_HPP
+#define PROTOZERO_PBF_MESSAGE_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file pbf_message.hpp
+ *
+ * @brief Contains the pbf_message class.
+ */
+
+#include <type_traits>
+
+#include <protozero/pbf_reader.hpp>
+#include <protozero/types.hpp>
+
+namespace protozero {
+
+/**
+ * This class represents a protobuf message. Either a top-level message or
+ * a nested sub-message. Top-level messages can be created from any buffer
+ * with a pointer and length:
+ *
+ * @code
+ *    enum class Message : protozero::pbf_tag_type {
+ *       ...
+ *    };
+ *
+ *    std::string buffer;
+ *    // fill buffer...
+ *    pbf_message<Message> message(buffer.data(), buffer.size());
+ * @endcode
+ *
+ * Sub-messages are created using get_message():
+ *
+ * @code
+ *    enum class SubMessage : protozero::pbf_tag_type {
+ *       ...
+ *    };
+ *
+ *    pbf_message<Message> message(...);
+ *    message.next();
+ *    pbf_message<SubMessage> submessage = message.get_message();
+ * @endcode
+ *
+ * All methods of the pbf_message class except get_bytes() and get_string()
+ * provide the strong exception guarantee, ie they either succeed or do not
+ * change the pbf_message object they are called on. Use the get_data() method
+ * instead of get_bytes() or get_string(), if you need this guarantee.
+ *
+ * This template class is based on the pbf_reader class and has all the same
+ * methods. The difference is that whereever the pbf_reader class takes an
+ * integer tag, this template class takes a tag of the template type T.
+ *
+ * Read the tutorial to understand how this class is used.
+ */
+template <typename T>
+class pbf_message : public pbf_reader {
+
+    static_assert(std::is_same<pbf_tag_type, typename std::underlying_type<T>::type>::value, "T must be enum with underlying type protozero::pbf_tag_type");
+
+public:
+
+    using enum_type = T;
+
+    template <typename... Args>
+    pbf_message(Args&&... args) noexcept :
+        pbf_reader(std::forward<Args>(args)...) {
+    }
+
+    inline bool next() {
+        return pbf_reader::next();
+    }
+
+    inline bool next(T tag) {
+        return pbf_reader::next(pbf_tag_type(tag));
+    }
+
+    inline T tag() const noexcept {
+        return T(pbf_reader::tag());
+    }
+
+};
+
+} // end namespace protozero
+
+#endif // PROTOZERO_PBF_MESSAGE_HPP

protozero/pbf_writer.hpp

@@ -0,0 +1,837 @@
+#ifndef PROTOZERO_PBF_WRITER_HPP
+#define PROTOZERO_PBF_WRITER_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file pbf_writer.hpp
+ *
+ * @brief Contains the pbf_writer class.
+ */
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <iterator>
+#include <limits>
+#include <string>
+
+#include <protozero/config.hpp>
+#include <protozero/types.hpp>
+#include <protozero/varint.hpp>
+
+#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
+# include <protozero/byteswap.hpp>
+#endif
+
+namespace protozero {
+
+namespace detail {
+
+    template <typename T> class packed_field_varint;
+    template <typename T> class packed_field_svarint;
+    template <typename T> class packed_field_fixed;
+
+} // end namespace detail
+
+/**
+ * The pbf_writer is used to write PBF formatted messages into a buffer.
+ *
+ * Almost all methods in this class can throw an std::bad_alloc exception if
+ * the std::string used as a buffer wants to resize.
+ */
+class pbf_writer {
+
+    // A pointer to a string buffer holding the data already written to the
+    // PBF message. For default constructed writers or writers that have been
+    // rolled back, this is a nullptr.
+    std::string* m_data;
+
+    // A pointer to a parent writer object if this is a submessage. If this
+    // is a top-level writer, it is a nullptr.
+    pbf_writer* m_parent_writer;
+
+    // This is usually 0. If there is an open submessage, this is set in the
+    // parent to the rollback position, ie. the last position before the
+    // submessage was started. This is the position where the header of the
+    // submessage starts.
+    std::size_t m_rollback_pos = 0;
+
+    // This is usually 0. If there is an open submessage, this is set in the
+    // parent to the position where the data of the submessage is written to.
+    std::size_t m_pos = 0;
+
+    inline void add_varint(uint64_t value) {
+        protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage");
+        protozero_assert(m_data);
+        write_varint(std::back_inserter(*m_data), value);
+    }
+
+    inline void add_field(pbf_tag_type tag, pbf_wire_type type) {
+        protozero_assert(((tag > 0 && tag < 19000) || (tag > 19999 && tag <= ((1 << 29) - 1))) && "tag out of range");
+        uint32_t b = (tag << 3) | uint32_t(type);
+        add_varint(b);
+    }
+
+    inline void add_tagged_varint(pbf_tag_type tag, uint64_t value) {
+        add_field(tag, pbf_wire_type::varint);
+        add_varint(value);
+    }
+
+    template <typename T>
+    inline void add_fixed(T value) {
+        protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage");
+        protozero_assert(m_data);
+#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN
+        m_data->append(reinterpret_cast<const char*>(&value), sizeof(T));
+#else
+        auto size = m_data->size();
+        m_data->resize(size + sizeof(T));
+        byteswap<sizeof(T)>(reinterpret_cast<const char*>(&value), const_cast<char*>(m_data->data() + size));
+#endif
+    }
+
+    template <typename T, typename It>
+    inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::input_iterator_tag) {
+        if (first == last) {
+            return;
+        }
+
+        pbf_writer sw(*this, tag);
+
+        while (first != last) {
+            sw.add_fixed<T>(*first++);
+        }
+    }
+
+    template <typename T, typename It>
+    inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::forward_iterator_tag) {
+        if (first == last) {
+            return;
+        }
+
+        auto length = std::distance(first, last);
+        add_length_varint(tag, sizeof(T) * pbf_length_type(length));
+        reserve(sizeof(T) * std::size_t(length));
+
+        while (first != last) {
+            add_fixed<T>(*first++);
+        }
+    }
+
+    template <typename It>
+    inline void add_packed_varint(pbf_tag_type tag, It first, It last) {
+        if (first == last) {
+            return;
+        }
+
+        pbf_writer sw(*this, tag);
+
+        while (first != last) {
+            sw.add_varint(uint64_t(*first++));
+        }
+    }
+
+    template <typename It>
+    inline void add_packed_svarint(pbf_tag_type tag, It first, It last) {
+        if (first == last) {
+            return;
+        }
+
+        pbf_writer sw(*this, tag);
+
+        while (first != last) {
+            sw.add_varint(encode_zigzag64(*first++));
+        }
+    }
+
+    // The number of bytes to reserve for the varint holding the length of
+    // a length-delimited field. The length has to fit into pbf_length_type,
+    // and a varint needs 8 bit for every 7 bit.
+    static const int reserve_bytes = sizeof(pbf_length_type) * 8 / 7 + 1;
+
+    // If m_rollpack_pos is set to this special value, it means that when
+    // the submessage is closed, nothing needs to be done, because the length
+    // of the submessage has already been written correctly.
+    static const std::size_t size_is_known = std::numeric_limits<std::size_t>::max();
+
+    inline void open_submessage(pbf_tag_type tag, std::size_t size) {
+        protozero_assert(m_pos == 0);
+        protozero_assert(m_data);
+        if (size == 0) {
+            m_rollback_pos = m_data->size();
+            add_field(tag, pbf_wire_type::length_delimited);
+            m_data->append(std::size_t(reserve_bytes), '\0');
+        } else {
+            m_rollback_pos = size_is_known;
+            add_length_varint(tag, pbf_length_type(size));
+            reserve(size);
+        }
+        m_pos = m_data->size();
+    }
+
+    inline void rollback_submessage() {
+        protozero_assert(m_pos != 0);
+        protozero_assert(m_rollback_pos != size_is_known);
+        protozero_assert(m_data);
+        m_data->resize(m_rollback_pos);
+        m_pos = 0;
+    }
+
+    inline void commit_submessage() {
+        protozero_assert(m_pos != 0);
+        protozero_assert(m_rollback_pos != size_is_known);
+        protozero_assert(m_data);
+        auto length = pbf_length_type(m_data->size() - m_pos);
+
+        protozero_assert(m_data->size() >= m_pos - reserve_bytes);
+        auto n = write_varint(m_data->begin() + long(m_pos) - reserve_bytes, length);
+
+        m_data->erase(m_data->begin() + long(m_pos) - reserve_bytes + n, m_data->begin() + long(m_pos));
+        m_pos = 0;
+    }
+
+    inline void close_submessage() {
+        protozero_assert(m_data);
+        if (m_pos == 0 || m_rollback_pos == size_is_known) {
+            return;
+        }
+        if (m_data->size() - m_pos == 0) {
+            rollback_submessage();
+        } else {
+            commit_submessage();
+        }
+    }
+
+    inline void add_length_varint(pbf_tag_type tag, pbf_length_type length) {
+        add_field(tag, pbf_wire_type::length_delimited);
+        add_varint(length);
+    }
+
+public:
+
+    /**
+     * Create a writer using the given string as a data store. The pbf_writer
+     * stores a reference to that string and adds all data to it. The string
+     * doesn't have to be empty. The pbf_writer will just append data.
+     */
+    inline explicit pbf_writer(std::string& data) noexcept :
+        m_data(&data),
+        m_parent_writer(nullptr),
+        m_pos(0) {
+    }
+
+    /**
+     * Create a writer without a data store. In this form the writer can not
+     * be used!
+     */
+    inline pbf_writer() noexcept :
+        m_data(nullptr),
+        m_parent_writer(nullptr),
+        m_pos(0) {
+    }
+
+    /**
+     * Construct a pbf_writer for a submessage from the pbf_writer of the
+     * parent message.
+     *
+     * @param parent_writer The pbf_writer
+     * @param tag Tag (field number) of the field that will be written
+     * @param size Optional size of the submessage in bytes (use 0 for unknown).
+     *        Setting this allows some optimizations but is only possible in
+     *        a few very specific cases.
+     */
+    inline pbf_writer(pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size=0) :
+        m_data(parent_writer.m_data),
+        m_parent_writer(&parent_writer),
+        m_pos(0) {
+        m_parent_writer->open_submessage(tag, size);
+    }
+
+    /// A pbf_writer object can be copied
+    pbf_writer(const pbf_writer&) noexcept = default;
+
+    /// A pbf_writer object can be copied
+    pbf_writer& operator=(const pbf_writer&) noexcept = default;
+
+    /// A pbf_writer object can be moved
+    inline pbf_writer(pbf_writer&&) noexcept = default;
+
+    /// A pbf_writer object can be moved
+    inline pbf_writer& operator=(pbf_writer&&) noexcept = default;
+
+    inline ~pbf_writer() {
+        if (m_parent_writer) {
+            m_parent_writer->close_submessage();
+        }
+    }
+
+    /**
+     * Reserve size bytes in the underlying message store in addition to
+     * whatever the message store already holds. So unlike
+     * the `std::string::reserve()` method this is not an absolute size,
+     * but additional memory that should be reserved.
+     *
+     * @param size Number of bytes to reserve in underlying message store.
+     */
+    void reserve(std::size_t size) {
+        protozero_assert(m_data);
+        m_data->reserve(m_data->size() + size);
+    }
+
+    inline void rollback() {
+        protozero_assert(m_parent_writer && "you can't call rollback() on a pbf_writer without a parent");
+        protozero_assert(m_pos == 0 && "you can't call rollback() on a pbf_writer that has an open nested submessage");
+        m_parent_writer->rollback_submessage();
+        m_data = nullptr;
+    }
+
+    ///@{
+    /**
+     * @name Scalar field writer functions
+     */
+
+    /**
+     * Add "bool" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_bool(pbf_tag_type tag, bool value) {
+        add_field(tag, pbf_wire_type::varint);
+        protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage");
+        protozero_assert(m_data);
+        m_data->append(1, value);
+    }
+
+    /**
+     * Add "enum" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_enum(pbf_tag_type tag, int32_t value) {
+        add_tagged_varint(tag, uint64_t(value));
+    }
+
+    /**
+     * Add "int32" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_int32(pbf_tag_type tag, int32_t value) {
+        add_tagged_varint(tag, uint64_t(value));
+    }
+
+    /**
+     * Add "sint32" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_sint32(pbf_tag_type tag, int32_t value) {
+        add_tagged_varint(tag, encode_zigzag32(value));
+    }
+
+    /**
+     * Add "uint32" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_uint32(pbf_tag_type tag, uint32_t value) {
+        add_tagged_varint(tag, value);
+    }
+
+    /**
+     * Add "int64" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_int64(pbf_tag_type tag, int64_t value) {
+        add_tagged_varint(tag, uint64_t(value));
+    }
+
+    /**
+     * Add "sint64" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_sint64(pbf_tag_type tag, int64_t value) {
+        add_tagged_varint(tag, encode_zigzag64(value));
+    }
+
+    /**
+     * Add "uint64" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_uint64(pbf_tag_type tag, uint64_t value) {
+        add_tagged_varint(tag, value);
+    }
+
+    /**
+     * Add "fixed32" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_fixed32(pbf_tag_type tag, uint32_t value) {
+        add_field(tag, pbf_wire_type::fixed32);
+        add_fixed<uint32_t>(value);
+    }
+
+    /**
+     * Add "sfixed32" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_sfixed32(pbf_tag_type tag, int32_t value) {
+        add_field(tag, pbf_wire_type::fixed32);
+        add_fixed<int32_t>(value);
+    }
+
+    /**
+     * Add "fixed64" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_fixed64(pbf_tag_type tag, uint64_t value) {
+        add_field(tag, pbf_wire_type::fixed64);
+        add_fixed<uint64_t>(value);
+    }
+
+    /**
+     * Add "sfixed64" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_sfixed64(pbf_tag_type tag, int64_t value) {
+        add_field(tag, pbf_wire_type::fixed64);
+        add_fixed<int64_t>(value);
+    }
+
+    /**
+     * Add "float" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_float(pbf_tag_type tag, float value) {
+        add_field(tag, pbf_wire_type::fixed32);
+        add_fixed<float>(value);
+    }
+
+    /**
+     * Add "double" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_double(pbf_tag_type tag, double value) {
+        add_field(tag, pbf_wire_type::fixed64);
+        add_fixed<double>(value);
+    }
+
+    /**
+     * Add "bytes" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Pointer to value to be written
+     * @param size Number of bytes to be written
+     */
+    inline void add_bytes(pbf_tag_type tag, const char* value, std::size_t size) {
+        protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage");
+        protozero_assert(m_data);
+        protozero_assert(size <= std::numeric_limits<pbf_length_type>::max());
+        add_length_varint(tag, pbf_length_type(size));
+        m_data->append(value, size);
+    }
+
+    /**
+     * Add "bytes" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_bytes(pbf_tag_type tag, const std::string& value) {
+        add_bytes(tag, value.data(), value.size());
+    }
+
+    /**
+     * Add "string" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Pointer to value to be written
+     * @param size Number of bytes to be written
+     */
+    inline void add_string(pbf_tag_type tag, const char* value, std::size_t size) {
+        add_bytes(tag, value, size);
+    }
+
+    /**
+     * Add "string" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written
+     */
+    inline void add_string(pbf_tag_type tag, const std::string& value) {
+        add_bytes(tag, value.data(), value.size());
+    }
+
+    /**
+     * Add "string" field to data. Bytes from the value are written until
+     * a null byte is encountered. The null byte is not added.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Pointer to value to be written
+     */
+    inline void add_string(pbf_tag_type tag, const char* value) {
+        add_bytes(tag, value, std::strlen(value));
+    }
+
+    /**
+     * Add "message" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Pointer to message to be written
+     * @param size Length of the message
+     */
+    inline void add_message(pbf_tag_type tag, const char* value, std::size_t size) {
+        add_bytes(tag, value, size);
+    }
+
+    /**
+     * Add "message" field to data.
+     *
+     * @param tag Tag (field number) of the field
+     * @param value Value to be written. The value must be a complete message.
+     */
+    inline void add_message(pbf_tag_type tag, const std::string& value) {
+        add_bytes(tag, value.data(), value.size());
+    }
+
+    ///@}
+
+    ///@{
+    /**
+     * @name Repeated packed field writer functions
+     */
+
+    /**
+     * Add "repeated packed bool" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to bool.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_bool(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_varint(tag, first, last);
+    }
+
+    /**
+     * Add "repeated packed enum" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to int32_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_enum(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_varint(tag, first, last);
+    }
+
+    /**
+     * Add "repeated packed int32" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to int32_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_int32(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_varint(tag, first, last);
+    }
+
+    /**
+     * Add "repeated packed sint32" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to int32_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_sint32(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_svarint(tag, first, last);
+    }
+
+    /**
+     * Add "repeated packed uint32" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to uint32_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_uint32(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_varint(tag, first, last);
+    }
+
+    /**
+     * Add "repeated packed int64" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to int64_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_int64(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_varint(tag, first, last);
+    }
+
+    /**
+     * Add "repeated packed sint64" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to int64_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_sint64(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_svarint(tag, first, last);
+    }
+
+    /**
+     * Add "repeated packed uint64" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to uint64_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_uint64(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_varint(tag, first, last);
+    }
+
+    /**
+     * Add "repeated packed fixed32" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to uint32_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_fixed32(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_fixed<uint32_t, InputIterator>(tag, first, last,
+            typename std::iterator_traits<InputIterator>::iterator_category());
+    }
+
+    /**
+     * Add "repeated packed sfixed32" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to int32_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_sfixed32(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_fixed<int32_t, InputIterator>(tag, first, last,
+            typename std::iterator_traits<InputIterator>::iterator_category());
+    }
+
+    /**
+     * Add "repeated packed fixed64" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to uint64_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_fixed64(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_fixed<uint64_t, InputIterator>(tag, first, last,
+            typename std::iterator_traits<InputIterator>::iterator_category());
+    }
+
+    /**
+     * Add "repeated packed sfixed64" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to int64_t.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_sfixed64(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_fixed<int64_t, InputIterator>(tag, first, last,
+            typename std::iterator_traits<InputIterator>::iterator_category());
+    }
+
+    /**
+     * Add "repeated packed float" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to float.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_float(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_fixed<float, InputIterator>(tag, first, last,
+            typename std::iterator_traits<InputIterator>::iterator_category());
+    }
+
+    /**
+     * Add "repeated packed double" field to data.
+     *
+     * @tparam InputIterator An type satisfying the InputIterator concept.
+     *         Dereferencing the iterator must yield a type assignable to double.
+     * @param tag Tag (field number) of the field
+     * @param first Iterator pointing to the beginning of the data
+     * @param last Iterator pointing one past the end of data
+     */
+    template <typename InputIterator>
+    inline void add_packed_double(pbf_tag_type tag, InputIterator first, InputIterator last) {
+        add_packed_fixed<double, InputIterator>(tag, first, last,
+            typename std::iterator_traits<InputIterator>::iterator_category());
+    }
+
+    ///@}
+
+    template <typename T> friend class detail::packed_field_varint;
+    template <typename T> friend class detail::packed_field_svarint;
+    template <typename T> friend class detail::packed_field_fixed;
+
+}; // class pbf_writer
+
+namespace detail {
+
+    class packed_field {
+
+    protected:
+
+        pbf_writer m_writer;
+
+    public:
+
+        packed_field(pbf_writer& parent_writer, pbf_tag_type tag) :
+            m_writer(parent_writer, tag) {
+        }
+
+        packed_field(pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size) :
+            m_writer(parent_writer, tag, size) {
+        }
+
+        void rollback() {
+            m_writer.rollback();
+        }
+
+    }; // class packed_field
+
+    template <typename T>
+    class packed_field_fixed : public packed_field {
+
+    public:
+
+        packed_field_fixed(pbf_writer& parent_writer, pbf_tag_type tag) :
+            packed_field(parent_writer, tag) {
+        }
+
+        packed_field_fixed(pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size) :
+            packed_field(parent_writer, tag, size * sizeof(T)) {
+        }
+
+        void add_element(T value) {
+            m_writer.add_fixed<T>(value);
+        }
+
+    }; // class packed_field_fixed
+
+    template <typename T>
+    class packed_field_varint : public packed_field {
+
+    public:
+
+        packed_field_varint(pbf_writer& parent_writer, pbf_tag_type tag) :
+            packed_field(parent_writer, tag) {
+        }
+
+        void add_element(T value) {
+            m_writer.add_varint(uint64_t(value));
+        }
+
+    }; // class packed_field_varint
+
+    template <typename T>
+    class packed_field_svarint : public packed_field {
+
+    public:
+
+        packed_field_svarint(pbf_writer& parent_writer, pbf_tag_type tag) :
+            packed_field(parent_writer, tag) {
+        }
+
+        void add_element(T value) {
+            m_writer.add_varint(encode_zigzag64(value));
+        }
+
+    }; // class packed_field_svarint
+
+} // end namespace detail
+
+using packed_field_bool     = detail::packed_field_varint<bool>;
+using packed_field_enum     = detail::packed_field_varint<int32_t>;
+using packed_field_int32    = detail::packed_field_varint<int32_t>;
+using packed_field_sint32   = detail::packed_field_svarint<int32_t>;
+using packed_field_uint32   = detail::packed_field_varint<uint32_t>;
+using packed_field_int64    = detail::packed_field_varint<int64_t>;
+using packed_field_sint64   = detail::packed_field_svarint<int64_t>;
+using packed_field_uint64   = detail::packed_field_varint<uint64_t>;
+using packed_field_fixed32  = detail::packed_field_fixed<uint32_t>;
+using packed_field_sfixed32 = detail::packed_field_fixed<int32_t>;
+using packed_field_fixed64  = detail::packed_field_fixed<uint64_t>;
+using packed_field_sfixed64 = detail::packed_field_fixed<int64_t>;
+using packed_field_float    = detail::packed_field_fixed<float>;
+using packed_field_double   = detail::packed_field_fixed<double>;
+
+} // end namespace protozero
+
+#endif // PROTOZERO_PBF_WRITER_HPP

protozero/types.hpp

@@ -0,0 +1,49 @@
+#ifndef PROTOZERO_TYPES_HPP
+#define PROTOZERO_TYPES_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file types.hpp
+ *
+ * @brief Contains the declaration of low-level types used in the pbf format.
+ */
+
+#include <cstdint>
+
+namespace protozero {
+
+    /**
+     * The type used for field tags (field numbers).
+     */
+    typedef uint32_t pbf_tag_type;
+
+    /**
+     * The type used to encode type information.
+     * See the table on
+     *    https://developers.google.com/protocol-buffers/docs/encoding
+     */
+    enum class pbf_wire_type : uint32_t {
+        varint           = 0, // int32/64, uint32/64, sint32/64, bool, enum
+        fixed64          = 1, // fixed64, sfixed64, double
+        length_delimited = 2, // string, bytes, embedded messages,
+                              // packed repeated fields
+        fixed32          = 5, // fixed32, sfixed32, float
+        unknown          = 99 // used for default setting in this library
+    };
+
+    /**
+     * The type used for length values, such as the length of a field.
+     */
+    typedef uint32_t pbf_length_type;
+
+} // end namespace protozero
+
+#endif // PROTOZERO_TYPES_HPP

protozero/varint.hpp

@@ -0,0 +1,132 @@
+#ifndef PROTOZERO_VARINT_HPP
+#define PROTOZERO_VARINT_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file varint.hpp
+ *
+ * @brief Contains low-level varint and zigzag encoding and decoding functions.
+ */
+
+#include <cstdint>
+
+#include <protozero/exception.hpp>
+
+namespace protozero {
+
+/**
+ * The maximum length of a 64bit varint.
+ */
+constexpr const int8_t max_varint_length = sizeof(uint64_t) * 8 / 7 + 1;
+
+// from https://github.com/facebook/folly/blob/master/folly/Varint.h
+/**
+ * Decode a 64bit varint.
+ *
+ * Strong exception guarantee: if there is an exception the data pointer will
+ * not be changed.
+ *
+ * @param[in,out] data Pointer to pointer to the input data. After the function
+ *        returns this will point to the next data to be read.
+ * @param[in] end Pointer one past the end of the input data.
+ * @returns The decoded integer
+ * @throws varint_too_long_exception if the varint is longer then the maximum
+ *         length that would fit in a 64bit int. Usually this means your data
+ *         is corrupted or you are trying to read something as a varint that
+ *         isn't.
+ * @throws end_of_buffer_exception if the *end* of the buffer was reached
+ *         before the end of the varint.
+ */
+inline uint64_t decode_varint(const char** data, const char* end) {
+    const int8_t* begin = reinterpret_cast<const int8_t*>(*data);
+    const int8_t* iend = reinterpret_cast<const int8_t*>(end);
+    const int8_t* p = begin;
+    uint64_t val = 0;
+
+    if (iend - begin >= max_varint_length) {  // fast path
+        do {
+            int64_t b;
+            b = *p++; val  = uint64_t((b & 0x7f)      ); if (b >= 0) break;
+            b = *p++; val |= uint64_t((b & 0x7f) <<  7); if (b >= 0) break;
+            b = *p++; val |= uint64_t((b & 0x7f) << 14); if (b >= 0) break;
+            b = *p++; val |= uint64_t((b & 0x7f) << 21); if (b >= 0) break;
+            b = *p++; val |= uint64_t((b & 0x7f) << 28); if (b >= 0) break;
+            b = *p++; val |= uint64_t((b & 0x7f) << 35); if (b >= 0) break;
+            b = *p++; val |= uint64_t((b & 0x7f) << 42); if (b >= 0) break;
+            b = *p++; val |= uint64_t((b & 0x7f) << 49); if (b >= 0) break;
+            b = *p++; val |= uint64_t((b & 0x7f) << 56); if (b >= 0) break;
+            b = *p++; val |= uint64_t((b & 0x7f) << 63); if (b >= 0) break;
+            throw varint_too_long_exception();
+        } while (false);
+    } else {
+        int shift = 0;
+        while (p != iend && *p < 0) {
+            val |= uint64_t(*p++ & 0x7f) << shift;
+            shift += 7;
+        }
+        if (p == iend) {
+            throw end_of_buffer_exception();
+        }
+        val |= uint64_t(*p++) << shift;
+    }
+
+    *data = reinterpret_cast<const char*>(p);
+    return val;
+}
+
+/**
+ * Varint-encode a 64bit integer.
+ */
+template <typename OutputIterator>
+inline int write_varint(OutputIterator data, uint64_t value) {
+    int n=1;
+
+    while (value >= 0x80) {
+        *data++ = char((value & 0x7f) | 0x80);
+        value >>= 7;
+        ++n;
+    }
+    *data++ = char(value);
+
+    return n;
+}
+
+/**
+ * ZigZag encodes a 32 bit integer.
+ */
+inline uint32_t encode_zigzag32(int32_t value) noexcept {
+    return (static_cast<uint32_t>(value) << 1) ^ (static_cast<uint32_t>(value >> 31));
+}
+
+/**
+ * ZigZag encodes a 64 bit integer.
+ */
+inline uint64_t encode_zigzag64(int64_t value) noexcept {
+    return (static_cast<uint64_t>(value) << 1) ^ (static_cast<uint64_t>(value >> 63));
+}
+
+/**
+ * Decodes a 32 bit ZigZag-encoded integer.
+ */
+inline int32_t decode_zigzag32(uint32_t value) noexcept {
+    return int32_t(value >> 1) ^ -int32_t(value & 1);
+}
+
+/**
+ * Decodes a 64 bit ZigZag-encoded integer.
+ */
+inline int64_t decode_zigzag64(uint64_t value) noexcept {
+    return int64_t(value >> 1) ^ -int64_t(value & 1);
+}
+
+} // end namespace protozero
+
+#endif // PROTOZERO_VARINT_HPP

protozero/version.hpp

@@ -0,0 +1,22 @@
+#ifndef PROTOZERO_VERSION_HPP
+#define PROTOZERO_VERSION_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+#define PROTOZERO_VERSION_MAJOR 1
+#define PROTOZERO_VERSION_MINOR 3
+#define PROTOZERO_VERSION_PATCH 0
+
+#define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH)
+
+#define PROTOZERO_VERSION_STRING "1.3.0"
+
+
+#endif // PROTOZERO_VERSION_HPP

tile-join.cc

@@ -1,5 +1,6 @@
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 #include <sqlite3.h>
 #include <limits.h>
@@ -8,7 +9,7 @@
 #include <map>
 #include <zlib.h>
 #include <math.h>
-#include "vector_tile.pb.h"
+#include "mvt.hh"
 #include "tile.h"
 
 extern "C" {
@@ -26,99 +27,25 @@ struct stats {
 	double minlat, minlon, maxlat, maxlon;
 };
 
-// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
-inline bool is_compressed(std::string const &data) {
-	return data.size() > 2 && (((uint8_t) data[0] == 0x78 && (uint8_t) data[1] == 0x9C) || ((uint8_t) data[0] == 0x1F && (uint8_t) data[1] == 0x8B));
-}
-
-// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
-inline int decompress(std::string const &input, std::string &output) {
-	z_stream inflate_s;
-	inflate_s.zalloc = Z_NULL;
-	inflate_s.zfree = Z_NULL;
-	inflate_s.opaque = Z_NULL;
-	inflate_s.avail_in = 0;
-	inflate_s.next_in = Z_NULL;
-	if (inflateInit2(&inflate_s, 32 + 15) != Z_OK) {
-		fprintf(stderr, "error: %s\n", inflate_s.msg);
-	}
-	inflate_s.next_in = (Bytef *) input.data();
-	inflate_s.avail_in = input.size();
-	size_t length = 0;
-	do {
-		output.resize(length + 2 * input.size());
-		inflate_s.avail_out = 2 * input.size();
-		inflate_s.next_out = (Bytef *) (output.data() + length);
-		int ret = inflate(&inflate_s, Z_FINISH);
-		if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
-			fprintf(stderr, "error: %s\n", inflate_s.msg);
-			return 0;
-		}
-
-		length += (2 * input.size() - inflate_s.avail_out);
-	} while (inflate_s.avail_out == 0);
-	inflateEnd(&inflate_s);
-	output.resize(length);
-	return 1;
-}
-
-// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
-static inline int compress(std::string const &input, std::string &output) {
-	z_stream deflate_s;
-	deflate_s.zalloc = Z_NULL;
-	deflate_s.zfree = Z_NULL;
-	deflate_s.opaque = Z_NULL;
-	deflate_s.avail_in = 0;
-	deflate_s.next_in = Z_NULL;
-	deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
-	deflate_s.next_in = (Bytef *) input.data();
-	deflate_s.avail_in = input.size();
-	size_t length = 0;
-	do {
-		size_t increase = input.size() / 2 + 1024;
-		output.resize(length + increase);
-		deflate_s.avail_out = increase;
-		deflate_s.next_out = (Bytef *) (output.data() + length);
-		int ret = deflate(&deflate_s, Z_FINISH);
-		if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
-			return -1;
-		}
-		length += (increase - deflate_s.avail_out);
-	} while (deflate_s.avail_out == 0);
-	deflateEnd(&deflate_s);
-	output.resize(length);
-	return 0;
-}
-
 void handle(std::string message, int z, unsigned x, unsigned y, struct pool **file_keys, char ***layernames, int *nlayers, sqlite3 *outdb, std::vector<std::string> &header, std::map<std::string, std::vector<std::string> > &mapping, struct pool *exclude, int ifmatched) {
-	GOOGLE_PROTOBUF_VERIFY_VERSION;
+	mvt_tile tile;
-
+	mvt_tile outtile;
-	// https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp
-	mapnik::vector::tile tile;
-	mapnik::vector::tile outtile;
 	int features_added = 0;
 
-	if (is_compressed(message)) {
+	if (!tile.decode(message)) {
-		std::string uncompressed;
+		fprintf(stderr, "Couldn't decompress tile %d/%u/%u\n", z, x, y);
-		decompress(message, uncompressed);
-		if (!tile.ParseFromString(uncompressed)) {
-			fprintf(stderr, "Couldn't decompress tile %d/%u/%u\n", z, x, y);
-			exit(EXIT_FAILURE);
-		}
-	} else if (!tile.ParseFromString(message)) {
-		fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y);
 		exit(EXIT_FAILURE);
 	}
 
-	for (int l = 0; l < tile.layers_size(); l++) {
+	for (size_t l = 0; l < tile.layers.size(); l++) {
-		mapnik::vector::tile_layer layer = tile.layers(l);
+		mvt_layer &layer = tile.layers[l];
-		mapnik::vector::tile_layer *outlayer = outtile.add_layers();
+		mvt_layer outlayer;
 
-		outlayer->set_name(layer.name());
+		outlayer.name = layer.name;
-		outlayer->set_version(layer.version());
+		outlayer.version = layer.version;
-		outlayer->set_extent(layer.extent());
+		outlayer.extent = layer.extent;
 
-		const char *ln = layer.name().c_str();
+		const char *ln = layer.name.c_str();
 
 		int ll;
 		for (ll = 0; ll < *nlayers; ll++) {
@@ -148,50 +75,46 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi
 			*nlayers = ll + 1;
 		}
 
-		struct pool keys, values;
+		for (size_t f = 0; f < layer.features.size(); f++) {
-		pool_init(&keys, 0);
+			mvt_feature feat = layer.features[f];
-		pool_init(&values, 0);
+			mvt_feature outfeature;
-
-		for (int f = 0; f < layer.features_size(); f++) {
-			mapnik::vector::tile_feature feat = layer.features(f);
-			std::vector<int> feature_tags;
 			int matched = 0;
 
-			for (int t = 0; t + 1 < feat.tags_size(); t += 2) {
+			for (int t = 0; t + 1 < feat.tags.size(); t += 2) {
-				const char *key = layer.keys(feat.tags(t)).c_str();
+				const char *key = layer.keys[feat.tags[t]].c_str();
-				mapnik::vector::tile_value const &val = layer.values(feat.tags(t + 1));
+				mvt_value &val = layer.values[feat.tags[t + 1]];
 				char *value;
 				int type = -1;
 
-				if (val.has_string_value()) {
+				if (val.type == mvt_string) {
-					value = strdup(val.string_value().c_str());
+					value = strdup(val.string_value.c_str());
 					if (value == NULL) {
 						perror("Out of memory");
 						exit(EXIT_FAILURE);
 					}
 					type = VT_STRING;
-				} else if (val.has_int_value()) {
+				} else if (val.type == mvt_int) {
-					if (asprintf(&value, "%lld", (long long) val.int_value()) >= 0) {
+					if (asprintf(&value, "%lld", (long long) val.numeric_value.int_value) >= 0) {
 						type = VT_NUMBER;
 					}
-				} else if (val.has_double_value()) {
+				} else if (val.type == mvt_double) {
-					if (asprintf(&value, "%g", val.double_value()) >= 0) {
+					if (asprintf(&value, "%g", val.numeric_value.double_value) >= 0) {
 						type = VT_NUMBER;
 					}
-				} else if (val.has_float_value()) {
+				} else if (val.type == mvt_float) {
-					if (asprintf(&value, "%g", val.float_value()) >= 0) {
+					if (asprintf(&value, "%g", val.numeric_value.float_value) >= 0) {
 						type = VT_NUMBER;
 					}
-				} else if (val.has_bool_value()) {
+				} else if (val.type == mvt_bool) {
-					if (asprintf(&value, "%s", val.bool_value() ? "true" : "false") >= 0) {
+					if (asprintf(&value, "%s", val.numeric_value.bool_value ? "true" : "false") >= 0) {
 						type = VT_BOOLEAN;
 					}
-				} else if (val.has_sint_value()) {
+				} else if (val.type == mvt_sint) {
-					if (asprintf(&value, "%lld", (long long) val.sint_value()) >= 0) {
+					if (asprintf(&value, "%lld", (long long) val.numeric_value.sint_value) >= 0) {
 						type = VT_NUMBER;
 					}
-				} else if (val.has_uint_value()) {
+				} else if (val.type == mvt_uint) {
-					if (asprintf(&value, "%llu", (long long) val.uint_value()) >= 0) {
+					if (asprintf(&value, "%llu", (long long) val.numeric_value.uint_value) >= 0) {
 						type = VT_NUMBER;
 					}
 				} else {
@@ -212,32 +135,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi
 						pool(&((*file_keys)[ll]), copy, type);
 					}
 
-					struct pool_val *k, *v;
+					outlayer.tag(outfeature, layer.keys[feat.tags[t]], val);
-
-					if (is_pooled(&keys, key, VT_STRING)) {
-						k = pool(&keys, key, VT_STRING);
-					} else {
-						char *copy = strdup(key);
-						if (copy == NULL) {
-							perror("Out of memory");
-							exit(EXIT_FAILURE);
-						}
-						k = pool(&keys, copy, VT_STRING);
-					}
-
-					if (is_pooled(&values, value, type)) {
-						v = pool(&values, value, type);
-					} else {
-						char *copy = strdup(value);
-						if (copy == NULL) {
-							perror("Out of memory");
-							exit(EXIT_FAILURE);
-						}
-						v = pool(&values, copy, type);
-					}
-
-					feature_tags.push_back(k->n);
-					feature_tags.push_back(v->n);
 				}
 
 				if (header.size() > 0 && strcmp(key, header[0].c_str()) == 0) {
@@ -261,7 +159,6 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi
 							}
 
 							const char *sjoinkey = joinkey.c_str();
-							const char *sjoinval = joinval.c_str();
 
 							if (!is_pooled(exclude, sjoinkey, VT_STRING)) {
 								if (!is_pooled(&((*file_keys)[ll]), sjoinkey, type)) {
@@ -273,32 +170,16 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi
 									pool(&((*file_keys)[ll]), copy, type);
 								}
 
-								struct pool_val *k, *v;
+								mvt_value outval;
-
+								if (type == VT_STRING) {
-								if (is_pooled(&keys, sjoinkey, VT_STRING)) {
+									outval.type = mvt_string;
-									k = pool(&keys, sjoinkey, VT_STRING);
+									outval.string_value = joinval;
 								} else {
-									char *copy = strdup(sjoinkey);
+									outval.type = mvt_double;
-									if (copy == NULL) {
+									outval.numeric_value.double_value = atof(joinval.c_str());
-										perror("Out of memory");
-										exit(EXIT_FAILURE);
-									}
-									k = pool(&keys, copy, VT_STRING);
 								}
 
-								if (is_pooled(&values, sjoinval, type)) {
+								outlayer.tag(outfeature, joinkey, outval);
-									v = pool(&values, sjoinval, type);
-								} else {
-									char *copy = strdup(sjoinval);
-									if (copy == NULL) {
-										perror("Out of memory");
-										exit(EXIT_FAILURE);
-									}
-									v = pool(&values, copy, type);
-								}
-
-								feature_tags.push_back(k->n);
-								feature_tags.push_back(v->n);
 							}
 						}
 					}
@@ -308,50 +189,22 @@ void handle(std::string message, int z, unsigned x, unsigned y, struct pool **fi
 			}
 
 			if (matched || !ifmatched) {
-				mapnik::vector::tile_feature *outfeature = outlayer->add_features();
+				outfeature.type = feat.type;
-				outfeature->set_type(feat.type());
+				outfeature.geometry = feat.geometry;
-
-				for (int g = 0; g < feat.geometry_size(); g++) {
-					outfeature->add_geometry(feat.geometry(g));
-				}
-
-				for (size_t i = 0; i < feature_tags.size(); i++) {
-					outfeature->add_tags(feature_tags[i]);
-				}
 
 				features_added++;
+				outlayer.features.push_back(outfeature);
 			}
 		}
 
-		struct pool_val *pv;
+		outtile.layers.push_back(outlayer);
-		for (pv = keys.head; pv != NULL; pv = pv->next) {
-			outlayer->add_keys(pv->s, strlen(pv->s));
-		}
-		for (pv = values.head; pv != NULL; pv = pv->next) {
-			mapnik::vector::tile_value *tv = outlayer->add_values();
-
-			if (pv->type == VT_NUMBER) {
-				tv->set_double_value(atof(pv->s));
-			} else if (pv->type == VT_BOOLEAN) {
-				tv->set_bool_value(pv->s[0] == 't');
-			} else {
-				tv->set_string_value(pv->s);
-			}
-		}
-
-		pool_free_strings(&keys);
-		pool_free_strings(&values);
 	}
 
 	if (features_added == 0) {
 		return;
 	}
 
-	std::string s;
+	std::string compressed = outtile.encode();
-	std::string compressed;
-
-	outtile.SerializeToString(&s);
-	compress(s, compressed);
 
 	if (compressed.size() > 500000) {
 		fprintf(stderr, "Tile %d/%u/%u size is %lld, >500000. Skipping this tile\n.", z, x, y, (long long) compressed.size());

tile.cc

@@ -8,6 +8,7 @@
 #include <algorithm>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 #include <limits.h>
 #include <zlib.h>
@@ -18,7 +19,7 @@
 #include <sqlite3.h>
 #include <pthread.h>
 #include <errno.h>
-#include "vector_tile.pb.h"
+#include "mvt.hh"
 #include "geometry.hh"
 
 extern "C" {
@@ -37,104 +38,36 @@ extern "C" {
 pthread_mutex_t db_lock = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_t var_lock = PTHREAD_MUTEX_INITIALIZER;
 
-// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
+std::vector<mvt_geometry> to_feature(drawvec &geom) {
-static inline int compress(std::string const &input, std::string &output) {
+	std::vector<mvt_geometry> out;
-	z_stream deflate_s;
+
-	deflate_s.zalloc = Z_NULL;
+	for (size_t i = 0; i < geom.size(); i++) {
-	deflate_s.zfree = Z_NULL;
+		out.push_back(mvt_geometry(geom[i].op, geom[i].x, geom[i].y));
-	deflate_s.opaque = Z_NULL;
+	}
-	deflate_s.avail_in = 0;
+
-	deflate_s.next_in = Z_NULL;
+	return out;
-	deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
-	deflate_s.next_in = (Bytef *) input.data();
-	deflate_s.avail_in = input.size();
-	size_t length = 0;
-	do {
-		size_t increase = input.size() / 2 + 1024;
-		output.resize(length + increase);
-		deflate_s.avail_out = increase;
-		deflate_s.next_out = (Bytef *) (output.data() + length);
-		int ret = deflate(&deflate_s, Z_FINISH);
-		if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
-			return -1;
-		}
-		length += (increase - deflate_s.avail_out);
-	} while (deflate_s.avail_out == 0);
-	deflateEnd(&deflate_s);
-	output.resize(length);
-	return 0;
 }
 
-int to_feature(drawvec &geom, mapnik::vector::tile_feature *feature) {
+bool draws_something(drawvec &geom) {
-	int px = 0, py = 0;
+	for (size_t i = 1; i < geom.size(); i++) {
-	int cmd_idx = -1;
+		if (geom[i].op == VT_LINETO && (geom[i].x != geom[i - 1].x || geom[i].y != geom[i - 1].y)) {
-	int cmd = -1;
+			return true;
-	int length = 0;
-	int drew = 0;
-	int i;
-
-	int n = geom.size();
-	for (i = 0; i < n; i++) {
-		int op = geom[i].op;
-
-		if (op != cmd) {
-			if (cmd_idx >= 0) {
-				if (feature != NULL) {
-					feature->set_geometry(cmd_idx, (length << CMD_BITS) | (cmd & ((1 << CMD_BITS) - 1)));
-				}
-			}
-
-			cmd = op;
-			length = 0;
-
-			if (feature != NULL) {
-				cmd_idx = feature->geometry_size();
-				feature->add_geometry(0);
-			}
-		}
-
-		if (op == VT_MOVETO || op == VT_LINETO) {
-			long long wwx = geom[i].x;
-			long long wwy = geom[i].y;
-
-			int dx = wwx - px;
-			int dy = wwy - py;
-
-			if (feature != NULL) {
-				feature->add_geometry((dx << 1) ^ (dx >> 31));
-				feature->add_geometry((dy << 1) ^ (dy >> 31));
-			}
-
-			px = wwx;
-			py = wwy;
-			length++;
-
-			if (op == VT_LINETO && (dx != 0 || dy != 0)) {
-				drew = 1;
-			}
-		} else if (op == VT_CLOSEPATH) {
-			length++;
-		} else {
-			fprintf(stderr, "\nInternal error: corrupted geometry\n");
-			exit(EXIT_FAILURE);
 		}
 	}
 
-	if (cmd_idx >= 0) {
+	return false;
-		if (feature != NULL) {
-			feature->set_geometry(cmd_idx, (length << CMD_BITS) | (cmd & ((1 << CMD_BITS) - 1)));
-		}
-	}
-
-	return drew;
 }
 
+int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char *stringpool2);
 int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2);
+static int is_integer(const char *s, long long *v);
 
 struct coalesce {
 	int type;
 	drawvec geom;
-	std::vector<int> meta;
+	int m;
+	char *meta;
+	char *stringpool;
 	unsigned long long index;
 	unsigned long long index2;
 	bool coalesced;
@@ -165,21 +98,10 @@ int coalcmp(const void *v1, const void *v2) {
 		return cmp;
 	}
 
-	for (size_t i = 0; i < c1->meta.size() && i < c2->meta.size(); i++) {
+	char *m1 = c1->meta;
-		cmp = c1->meta[i] - c2->meta[i];
+	char *m2 = c2->meta;
 
-		if (cmp != 0) {
+	return metacmp(c1->m, &m1, c1->stringpool, c2->m, &m2, c2->stringpool);
-			return cmp;
-		}
-	}
-
-	if (c1->meta.size() < c2->meta.size()) {
-		return -1;
-	} else if (c1->meta.size() > c2->meta.size()) {
-		return 1;
-	} else {
-		return 0;
-	}
 }
 
 int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2) {
@@ -202,38 +124,65 @@ int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2) {
 	return cmp;
 }
 
-struct pool_val *retrieve_string(char **f, struct pool *p, char *stringpool) {
+mvt_value retrieve_string(char **f, char *stringpool, int *otype) {
-	struct pool_val *ret;
 	long long off;
-
 	deserialize_long_long(f, &off);
-	ret = pool(p, stringpool + off + 1, stringpool[off]);
 
-	return ret;
+	int type = stringpool[off];
+	char *s = stringpool + off + 1;
+
+	if (otype != NULL) {
+		*otype = type;
+	}
+
+	mvt_value tv;
+	if (type == VT_NUMBER) {
+		long long v;
+		if (is_integer(s, &v)) {
+			if (v >= 0) {
+				tv.type = mvt_int;
+				tv.numeric_value.int_value = v;
+			} else {
+				tv.type = mvt_sint;
+				tv.numeric_value.sint_value = v;
+			}
+		} else {
+			tv.type = mvt_double;
+			tv.numeric_value.double_value = atof(s);
+		}
+	} else if (type == VT_BOOLEAN) {
+		tv.type = mvt_bool;
+		tv.numeric_value.bool_value = (s[0] == 't');
+	} else {
+		tv.type = mvt_string;
+		tv.string_value = s;
+	}
+
+	return tv;
 }
 
-void decode_meta(int m, char **meta, char *stringpool, struct pool *keys, struct pool *values, struct pool *file_keys, std::vector<int> *intmeta) {
+void decode_meta(int m, char **meta, char *stringpool, mvt_layer &layer, mvt_feature &feature, struct pool *file_keys) {
 	int i;
 	for (i = 0; i < m; i++) {
-		struct pool_val *key = retrieve_string(meta, keys, stringpool);
+		int otype;
-		struct pool_val *value = retrieve_string(meta, values, stringpool);
+		mvt_value key = retrieve_string(meta, stringpool, NULL);
+		mvt_value value = retrieve_string(meta, stringpool, &otype);
 
-		intmeta->push_back(key->n);
+		layer.tag(feature, key.string_value, value);
-		intmeta->push_back(value->n);
 
-		if (!is_pooled(file_keys, key->s, value->type)) {
+		if (!is_pooled(file_keys, key.string_value.c_str(), otype)) {
 			if (pthread_mutex_lock(&var_lock) != 0) {
 				perror("pthread_mutex_lock");
 				exit(EXIT_FAILURE);
 			}
 
 			// Dup to retain after munmap
-			char *copy = strdup(key->s);
+			char *copy = strdup(key.string_value.c_str());
 			if (copy == NULL) {
 				perror("Out of memory");
 				exit(EXIT_FAILURE);
 			}
-			pool(file_keys, copy, value->type);
+			pool(file_keys, copy, otype);
 
 			if (pthread_mutex_unlock(&var_lock) != 0) {
 				perror("pthread_mutex_unlock");
@@ -243,6 +192,51 @@ void decode_meta(int m, char **meta, char *stringpool, struct pool *keys, struct
 	}
 }
 
+int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char *stringpool2) {
+	// XXX
+	// Ideally this would make identical features compare the same lexically
+	// even if their attributes were declared in different orders in different instances.
+	// In practice, this is probably good enough to put "identical" features together.
+
+	int i;
+	for (i = 0; i < m1 && i < m2; i++) {
+		mvt_value key1 = retrieve_string(meta1, stringpool1, NULL);
+		mvt_value key2 = retrieve_string(meta2, stringpool2, NULL);
+
+		if (key1.string_value < key2.string_value) {
+			return -1;
+		} else if (key1.string_value > key2.string_value) {
+			return 1;
+		}
+
+		long long off1;
+		deserialize_long_long(meta1, &off1);
+		int type1 = stringpool1[off1];
+		char *s1 = stringpool1 + off1 + 1;
+
+		long long off2;
+		deserialize_long_long(meta2, &off2);
+		int type2 = stringpool2[off2];
+		char *s2 = stringpool2 + off2 + 1;
+
+		if (type1 != type2) {
+			return type1 - type2;
+		}
+		int cmp = strcmp(s1, s2);
+		if (s1 != s2) {
+			return cmp;
+		}
+	}
+
+	if (m1 < m2) {
+		return -1;
+	} else if (m1 > m2) {
+		return 1;
+	} else {
+		return 0;
+	}
+}
+
 static int is_integer(const char *s, long long *v) {
 	errno = 0;
 	char *endptr;
@@ -274,75 +268,6 @@ static int is_integer(const char *s, long long *v) {
 	return 1;
 }
 
-mapnik::vector::tile create_tile(char **layernames, int line_detail, std::vector<std::vector<coalesce> > &features, long long *count, struct pool **keys, struct pool **values, int nlayers) {
-	mapnik::vector::tile tile;
-
-	int i;
-	for (i = 0; i < nlayers; i++) {
-		if (features[i].size() == 0) {
-			continue;
-		}
-
-		mapnik::vector::tile_layer *layer = tile.add_layers();
-
-		layer->set_name(layernames[i]);
-		layer->set_version(1);
-		layer->set_extent(1 << line_detail);
-
-		for (size_t x = 0; x < features[i].size(); x++) {
-			if (features[i][x].type == VT_LINE || features[i][x].type == VT_POLYGON) {
-				features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type, 0);
-			}
-
-			mapnik::vector::tile_feature *feature = layer->add_features();
-
-			if (features[i][x].type == VT_POINT) {
-				feature->set_type(mapnik::vector::tile::Point);
-			} else if (features[i][x].type == VT_LINE) {
-				feature->set_type(mapnik::vector::tile::LineString);
-			} else if (features[i][x].type == VT_POLYGON) {
-				feature->set_type(mapnik::vector::tile::Polygon);
-			} else {
-				feature->set_type(mapnik::vector::tile::Unknown);
-			}
-
-			to_feature(features[i][x].geom, feature);
-			*count += features[i][x].geom.size();
-
-			for (size_t y = 0; y < features[i][x].meta.size(); y++) {
-				feature->add_tags(features[i][x].meta[y]);
-			}
-		}
-
-		struct pool_val *pv;
-		for (pv = keys[i]->head; pv != NULL; pv = pv->next) {
-			layer->add_keys(pv->s, strlen(pv->s));
-		}
-		for (pv = values[i]->head; pv != NULL; pv = pv->next) {
-			mapnik::vector::tile_value *tv = layer->add_values();
-
-			if (pv->type == VT_NUMBER) {
-				long long v;
-				if (is_integer(pv->s, &v)) {
-					if (v >= 0) {
-						tv->set_int_value(v);
-					} else {
-						tv->set_sint_value(v);
-					}
-				} else {
-					tv->set_double_value(atof(pv->s));
-				}
-			} else if (pv->type == VT_BOOLEAN) {
-				tv->set_bool_value(pv->s[0] == 't');
-			} else {
-				tv->set_string_value(pv->s);
-			}
-		}
-	}
-
-	return tile;
-}
-
 struct sll {
 	char *name;
 	long long val;
@@ -633,19 +558,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 	// This only loops if the tile data didn't fit, in which case the detail
 	// goes down and the progress indicator goes backward for the next try.
 	for (line_detail = detail; line_detail >= min_detail || line_detail == detail; line_detail--, oprogress = 0) {
-		GOOGLE_PROTOBUF_VERIFY_VERSION;
-
-		struct pool keys1[nlayers], values1[nlayers];
-		struct pool *keys[nlayers], *values[nlayers];
-		int i;
-		for (i = 0; i < nlayers; i++) {
-			pool_init(&keys1[i], 0);
-			pool_init(&values1[i], 0);
-
-			keys[i] = &keys1[i];
-			values[i] = &values1[i];
-		}
-
 		long long count = 0;
 		double accum_area = 0;
 
@@ -666,7 +578,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 
 		std::vector<struct partial> partials;
 		std::vector<std::vector<coalesce> > features;
-		for (i = 0; i < nlayers; i++) {
+		for (size_t i = 0; i < nlayers; i++) {
 			features.push_back(std::vector<coalesce>());
 		}
 
@@ -912,21 +824,18 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 			}
 		}
 
-		// This is serial because decode_meta() unifies duplicates
 		for (size_t i = 0; i < partials.size(); i++) {
 			std::vector<drawvec> geoms = partials[i].geoms;
 			partials[i].geoms.clear();  // avoid keeping two copies in memory
 			long long layer = partials[i].layer;
 			signed char t = partials[i].t;
-			int segment = partials[i].segment;
 			long long original_seq = partials[i].original_seq;
 
 			// A complex polygon may have been split up into multiple geometries.
 			// Break them out into multiple features if necessary.
 			for (size_t j = 0; j < geoms.size(); j++) {
-				if (t == VT_POINT || to_feature(geoms[j], NULL)) {
+				if (t == VT_POINT || draws_something(geoms[j])) {
 					struct coalesce c;
-					char *meta = partials[i].meta;
 
 					c.type = t;
 					c.index = partials[i].index;
@@ -934,8 +843,10 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 					c.geom = geoms[j];
 					c.coalesced = false;
 					c.original_seq = original_seq;
+					c.m = partials[i].m;
+					c.meta = partials[i].meta;
+					c.stringpool = stringpool + pool_off[partials[i].segment];
 
-					decode_meta(partials[i].m, &meta, stringpool + pool_off[segment], keys[layer], values[layer], file_keys[layer], &c.meta);
 					features[layer].push_back(c);
 				}
 			}
@@ -1003,6 +914,35 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 			}
 		}
 
+		mvt_tile tile;
+
+		for (size_t j = 0; j < features.size(); j++) {
+			mvt_layer layer;
+
+			layer.name = layernames[j];
+			layer.version = 2;
+			layer.extent = 1 << line_detail;
+
+			for (size_t x = 0; x < features[j].size(); x++) {
+				mvt_feature feature;
+
+				if (features[j][x].type == VT_LINE || features[j][x].type == VT_POLYGON) {
+					features[j][x].geom = remove_noop(features[j][x].geom, features[j][x].type, 0);
+				}
+
+				feature.type = features[j][x].type;
+				feature.geometry = to_feature(features[j][x].geom);
+				count += features[j][x].geom.size();
+
+				decode_meta(features[j][x].m, &features[j][x].meta, features[j][x].stringpool, layer, feature, file_keys[j]);
+				layer.features.push_back(feature);
+			}
+
+			if (layer.features.size() > 0) {
+				tile.layers.push_back(layer);
+			}
+		}
+
 		if (z == 0 && unclipped_features < original_features / 2) {
 			fprintf(stderr, "\n\nMore than half the features were clipped away at zoom level 0.\n");
 			fprintf(stderr, "Is your data in the wrong projection? It should be in WGS84/EPSG:4326.\n");
@@ -1020,19 +960,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				return -1;
 			}
 
-			mapnik::vector::tile tile = create_tile(layernames, line_detail, features, &count, keys, values, nlayers);
+			std::string compressed = tile.encode();
-
-			int i;
-			for (i = 0; i < nlayers; i++) {
-				pool_free(&keys1[i]);
-				pool_free(&values1[i]);
-			}
-
-			std::string s;
-			std::string compressed;
-
-			tile.SerializeToString(&s);
-			compress(s, compressed);
 
 			if (compressed.size() > 500000 && !prevent[P_KILOBYTE_LIMIT]) {
 				if (!quiet) {
@@ -1065,12 +993,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				return count;
 			}
 		} else {
-			int i;
-			for (i = 0; i < nlayers; i++) {
-				pool_free(&keys1[i]);
-				pool_free(&values1[i]);
-			}
-
 			return count;
 		}
 	}

version.h

@@ -1 +1 @@
-#define VERSION "tippecanoe v1.9.15\n"
+#define VERSION "tippecanoe v1.9.16\n"