Factor out vector tile writing

decode.cc

@@ -96,6 +96,15 @@ void handle(std::string message, int z, unsigned x, unsigned y, int describe) {
 					printf(", ");
 				}
 
+				if (feat.tags[t] >= layer.keys.size()) {
+					fprintf(stderr, "Error: out of bounds feature key\n");
+					exit(EXIT_FAILURE);
+				}
+				if (feat.tags[t + 1] >= layer.values.size()) {
+					fprintf(stderr, "Error: out of bounds feature value\n");
+					exit(EXIT_FAILURE);
+				}
+
 				const char *key = layer.keys[feat.tags[t]].c_str();
 				pb_value const &val = layer.values[feat.tags[t + 1]];
 

protobuf.cc

@@ -1,3 +1,4 @@
+#include <stdio.h>
 #include <string>
 #include <vector>
 #include <zlib.h>
@@ -94,7 +95,7 @@ bool pb_decode(std::string &message, pb_tile &out) {
 		return false;
 	}
 
-	for (int l = 0; l < tile.layers_size(); l++) {
+	for (size_t l = 0; l < tile.layers_size(); l++) {
 		mapnik::vector::tile_layer layer = tile.layers(l);
 		pb_layer pbl;
 		pbl.extent = layer.extent();
@@ -134,7 +135,7 @@ bool pb_decode(std::string &message, pb_tile &out) {
 			pbl.values.push_back(pbv);
 		}
 
-		for (int f = 0; f < layer.features_size(); f++) {
+		for (size_t f = 0; f < layer.features_size(); f++) {
 			mapnik::vector::tile_feature feat = layer.features(f);
 			pb_feature pbf;
 			pbf.type = feat.type();
@@ -170,3 +171,119 @@ bool pb_decode(std::string &message, pb_tile &out) {
 
 	return true;
 }
+
+std::string pb_encode(pb_tile &in) {
+	GOOGLE_PROTOBUF_VERIFY_VERSION;
+
+	mapnik::vector::tile tile;
+
+	for (size_t i = 0; i < in.layers.size(); i++) {
+		mapnik::vector::tile_layer *layer = tile.add_layers();
+
+		layer->set_name(in.layers[i].name);
+		layer->set_version(in.layers[i].version);
+		layer->set_extent(in.layers[i].extent);
+
+		for (size_t k = 0; k < in.layers[i].keys.size(); k++) {
+			layer->add_keys(in.layers[i].keys[k]);
+		}
+		for (size_t v = 0; v < in.layers[i].values.size(); v++) {
+			mapnik::vector::tile_value *tv = layer->add_values();
+			pb_value &pbv = in.layers[i].values[v];
+
+			if (pbv.type == pb_string) {
+				tv->set_string_value(pbv.string_value);
+			} else if (pbv.type == pb_float) {
+				tv->set_float_value(pbv.numeric_value.float_value);
+			} else if (pbv.type == pb_double) {
+				tv->set_double_value(pbv.numeric_value.double_value);
+			} else if (pbv.type == pb_int) {
+				tv->set_int_value(pbv.numeric_value.int_value);
+			} else if (pbv.type == pb_uint) {
+				tv->set_uint_value(pbv.numeric_value.uint_value);
+			} else if (pbv.type == pb_sint) {
+				tv->set_sint_value(pbv.numeric_value.sint_value);
+			} else if (pbv.type == pb_bool) {
+				tv->set_bool_value(pbv.numeric_value.bool_value);
+			}
+		}
+
+		for (size_t f = 0; f < in.layers[i].features.size(); f++) {
+			mapnik::vector::tile_feature *feature = layer->add_features();
+			if (feature == NULL) {
+				perror("add feature");
+				exit(EXIT_FAILURE);
+			}
+
+			int type = in.layers[i].features[f].type;
+			if (type == pb_point) {
+				feature->set_type(mapnik::vector::tile::Point);
+			} else if (type == pb_linestring) {
+				feature->set_type(mapnik::vector::tile::LineString);
+			} else if (type == pb_polygon) {
+				feature->set_type(mapnik::vector::tile::Polygon);
+			} else {
+				fprintf(stderr, "Corrupt geometry type\n");
+				exit(EXIT_FAILURE);
+			}
+
+			for (size_t t = 0; t < in.layers[i].features[f].tags.size(); t++) {
+				feature->add_tags(in.layers[i].features[f].tags[t]);
+			}
+
+			int px = 0, py = 0;
+			int cmd_idx = -1;
+			int cmd = -1;
+			int length = 0;
+
+			std::vector<pb_geometry> &geom = in.layers[i].features[f].geometry;
+
+			for (size_t g = 0; g < geom.size(); g++) {
+				int op = geom[g].op;
+
+				if (op != cmd) {
+					if (cmd_idx >= 0) {
+						feature->set_geometry(cmd_idx, (length << 3) | (cmd & ((1 << 3) - 1)));
+					}
+
+					cmd = op;
+					length = 0;
+					cmd_idx = feature->geometry_size();
+					feature->add_geometry(0);
+				}
+
+				if (op == pb_moveto || op == pb_lineto) {
+					long long wwx = geom[g].x;
+					long long wwy = geom[g].y;
+
+					int dx = wwx - px;
+					int dy = wwy - py;
+
+					if (feature != NULL) {
+						feature->add_geometry((dx << 1) ^ (dx >> 31));
+						feature->add_geometry((dy << 1) ^ (dy >> 31));
+					}
+
+					px = wwx;
+					py = wwy;
+					length++;
+				} else if (op == pb_closepath) {
+					length++;
+				} else {
+					fprintf(stderr, "\nInternal error: corrupted geometry\n");
+					exit(EXIT_FAILURE);
+				}
+			}
+
+			if (cmd_idx >= 0) {
+				feature->set_geometry(cmd_idx, (length << 3) | (cmd & ((1 << 3) - 1)));
+			}
+		}
+	}
+
+	std::string s, compressed;
+	tile.SerializeToString(&s);
+	compress(s, compressed);
+
+	return compressed;
+}

protobuf.hh

@@ -60,3 +60,4 @@ struct pb_tile {
 };
 
 bool pb_decode(std::string &message, pb_tile &out);
+std::string pb_encode(pb_tile &in);

tile.cc

@@ -38,68 +38,24 @@ extern "C" {
 pthread_mutex_t db_lock = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_t var_lock = PTHREAD_MUTEX_INITIALIZER;
 
-int to_feature(drawvec &geom, mapnik::vector::tile_feature *feature) {
+std::vector<pb_geometry> to_feature(drawvec &geom) {
-	int px = 0, py = 0;
+	std::vector<pb_geometry> out;
-	int cmd_idx = -1;
-	int cmd = -1;
-	int length = 0;
-	int drew = 0;
-	int i;
 
-	int n = geom.size();
+	for (size_t i = 0; i < geom.size(); i++) {
-	for (i = 0; i < n; i++) {
+		out.push_back(pb_geometry(geom[i].op, geom[i].x, geom[i].y));
-		int op = geom[i].op;
+	}
 
-		if (op != cmd) {
+	return out;
-			if (cmd_idx >= 0) {
+}
-				if (feature != NULL) {
-					feature->set_geometry(cmd_idx, (length << CMD_BITS) | (cmd & ((1 << CMD_BITS) - 1)));
-				}
-			}
 
-			cmd = op;
+bool draws_something(drawvec &geom) {
-			length = 0;
+	for (size_t i = 1; i < geom.size(); i++) {
-
+		if (geom[i].op == VT_LINETO && (geom[i].x != geom[i - 1].x || geom[i].y != geom[i - 1].y)) {
-			if (feature != NULL) {
+			return true;
-				cmd_idx = feature->geometry_size();
-				feature->add_geometry(0);
-			}
-		}
-
-		if (op == VT_MOVETO || op == VT_LINETO) {
-			long long wwx = geom[i].x;
-			long long wwy = geom[i].y;
-
-			int dx = wwx - px;
-			int dy = wwy - py;
-
-			if (feature != NULL) {
-				feature->add_geometry((dx << 1) ^ (dx >> 31));
-				feature->add_geometry((dy << 1) ^ (dy >> 31));
-			}
-
-			px = wwx;
-			py = wwy;
-			length++;
-
-			if (op == VT_LINETO && (dx != 0 || dy != 0)) {
-				drew = 1;
-			}
-		} else if (op == VT_CLOSEPATH) {
-			length++;
-		} else {
-			fprintf(stderr, "\nInternal error: corrupted geometry\n");
-			exit(EXIT_FAILURE);
 		}
 	}
 
-	if (cmd_idx >= 0) {
+	return false;
-		if (feature != NULL) {
-			feature->set_geometry(cmd_idx, (length << CMD_BITS) | (cmd & ((1 << CMD_BITS) - 1)));
-		}
-	}
-
-	return drew;
 }
 
 int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2);
@@ -247,8 +203,8 @@ static int is_integer(const char *s, long long *v) {
 	return 1;
 }
 
-mapnik::vector::tile create_tile(char **layernames, int line_detail, std::vector<std::vector<coalesce> > &features, long long *count, struct pool **keys, struct pool **values, int nlayers) {
+pb_tile create_tile(char **layernames, int line_detail, std::vector<std::vector<coalesce> > &features, long long *count, struct pool **keys, struct pool **values, int nlayers) {
-	mapnik::vector::tile tile;
+	pb_tile tile;
 
 	int i;
 	for (i = 0; i < nlayers; i++) {
@@ -256,61 +212,63 @@ mapnik::vector::tile create_tile(char **layernames, int line_detail, std::vector
 			continue;
 		}
 
-		mapnik::vector::tile_layer *layer = tile.add_layers();
+		pb_layer layer;
 
-		layer->set_name(layernames[i]);
+		layer.name = layernames[i];
-		layer->set_version(1);
+		layer.version = 1;
-		layer->set_extent(1 << line_detail);
+		layer.extent = 1 << line_detail;
 
 		for (size_t x = 0; x < features[i].size(); x++) {
 			if (features[i][x].type == VT_LINE || features[i][x].type == VT_POLYGON) {
 				features[i][x].geom = remove_noop(features[i][x].geom, features[i][x].type, 0);
 			}
 
-			mapnik::vector::tile_feature *feature = layer->add_features();
+			pb_feature pbf;
+			pbf.type = features[i][x].type;
 
-			if (features[i][x].type == VT_POINT) {
+			pbf.geometry = to_feature(features[i][x].geom);
-				feature->set_type(mapnik::vector::tile::Point);
-			} else if (features[i][x].type == VT_LINE) {
-				feature->set_type(mapnik::vector::tile::LineString);
-			} else if (features[i][x].type == VT_POLYGON) {
-				feature->set_type(mapnik::vector::tile::Polygon);
-			} else {
-				feature->set_type(mapnik::vector::tile::Unknown);
-			}
-
-			to_feature(features[i][x].geom, feature);
 			*count += features[i][x].geom.size();
 
 			for (size_t y = 0; y < features[i][x].meta.size(); y++) {
-				feature->add_tags(features[i][x].meta[y]);
+				pbf.tags.push_back(features[i][x].meta[y]);
 			}
+
+			layer.features.push_back(pbf);
 		}
 
 		struct pool_val *pv;
 		for (pv = keys[i]->head; pv != NULL; pv = pv->next) {
-			layer->add_keys(pv->s, strlen(pv->s));
+			layer.keys.push_back(std::string(pv->s, strlen(pv->s)));
 		}
 		for (pv = values[i]->head; pv != NULL; pv = pv->next) {
-			mapnik::vector::tile_value *tv = layer->add_values();
+			pb_value tv;
 
 			if (pv->type == VT_NUMBER) {
 				long long v;
 				if (is_integer(pv->s, &v)) {
 					if (v >= 0) {
-						tv->set_int_value(v);
+						tv.type = pb_int;
+						tv.numeric_value.int_value = v;
 					} else {
-						tv->set_sint_value(v);
+						tv.type = pb_sint;
+						tv.numeric_value.sint_value = v;
 					}
 				} else {
-					tv->set_double_value(atof(pv->s));
+					tv.type = pb_double;
+					tv.numeric_value.double_value = atof(pv->s);
 				}
 			} else if (pv->type == VT_BOOLEAN) {
-				tv->set_bool_value(pv->s[0] == 't');
+				tv.type = pb_bool;
+				tv.numeric_value.bool_value = (pv->s[0] == 't');
 			} else {
-				tv->set_string_value(pv->s);
+				tv.type = pb_string;
+				tv.string_value = pv->s;
 			}
+
+			layer.values.push_back(tv);
 		}
+
+		tile.layers.push_back(layer);
 	}
 
 	return tile;
@@ -890,7 +848,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 			// A complex polygon may have been split up into multiple geometries.
 			// Break them out into multiple features if necessary.
 			for (size_t j = 0; j < geoms.size(); j++) {
-				if (t == VT_POINT || to_feature(geoms[j], NULL)) {
+				if (t == VT_POINT || draws_something(geoms[j])) {
 					struct coalesce c;
 					char *meta = partials[i].meta;
 
@@ -986,7 +944,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				return -1;
 			}
 
-			mapnik::vector::tile tile = create_tile(layernames, line_detail, features, &count, keys, values, nlayers);
+			pb_tile tile = create_tile(layernames, line_detail, features, &count, keys, values, nlayers);
 
 			int i;
 			for (i = 0; i < nlayers; i++) {
@@ -994,11 +952,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				pool_free(&values1[i]);
 			}
 
-			std::string s;
+			std::string compressed = pb_encode(tile);
-			std::string compressed;
-
-			tile.SerializeToString(&s);
-			compress(s, compressed);
 
 			if (compressed.size() > 500000 && !prevent[P_KILOBYTE_LIMIT]) {
 				if (!quiet) {