Factor out feature deserialization

geometry.cpp

@@ -18,7 +18,7 @@
 static int pnpoly(drawvec &vert, size_t start, size_t nvert, long long testx, long long testy);
 static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax);
 
-drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y) {
+drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y) {
 	drawvec out;
 
 	bbox[0] = LLONG_MAX;

geometry.hpp

@@ -53,7 +53,7 @@ struct draw {
 
 typedef std::vector<draw> drawvec;
 
-drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, int detail, long long *bbox, unsigned initial_x, unsigned initial_y);
+drawvec decode_geometry(FILE *meta, long long *geompos, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y);
 void to_tile_scale(drawvec &geom, int z, int detail);
 drawvec remove_noop(drawvec geom, int type, int shift);
 drawvec clip_point(drawvec &geom, int z, int detail, long long buffer);

serial.cpp

@@ -230,3 +230,83 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, c
 		serialize_byte(geomfile, sf->feature_minzoom, geompos, fname);
 	}
 }
+
+serial_feature deserialize_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
+	serial_feature sf;
+
+	deserialize_byte_io(geoms, &sf.t, geompos_in);
+	if (sf.t < 0) {
+		return sf;
+	}
+
+	deserialize_long_long_io(geoms, &sf.layer, geompos_in);
+
+	sf.seq = 0;
+	if (sf.layer & (1 << 5)) {
+		deserialize_long_long_io(geoms, &sf.seq, geompos_in);
+	}
+
+	sf.tippecanoe_minzoom = -1;
+	sf.tippecanoe_maxzoom = -1;
+	sf.id = 0;
+	sf.has_id = false;
+	if (sf.layer & (1 << 1)) {
+		deserialize_int_io(geoms, &sf.tippecanoe_minzoom, geompos_in);
+	}
+	if (sf.layer & (1 << 0)) {
+		deserialize_int_io(geoms, &sf.tippecanoe_maxzoom, geompos_in);
+	}
+	if (sf.layer & (1 << 2)) {
+		sf.has_id = true;
+		deserialize_ulong_long_io(geoms, &sf.id, geompos_in);
+	}
+
+	deserialize_int_io(geoms, &sf.segment, geompos_in);
+
+	sf.index = 0;
+	sf.extent = 0;
+
+	sf.geometry = decode_geometry(geoms, geompos_in, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]);
+	if (sf.layer & (1 << 4)) {
+		deserialize_ulong_long_io(geoms, &sf.index, geompos_in);
+	}
+	if (sf.layer & (1 << 3)) {
+		deserialize_long_long_io(geoms, &sf.extent, geompos_in);
+	}
+
+	sf.layer >>= 6;
+
+	sf.metapos = 0;
+	{
+		int m;
+		deserialize_int_io(geoms, &m, geompos_in);
+		sf.m = m;
+	}
+	if (sf.m != 0) {
+		deserialize_long_long_io(geoms, &sf.metapos, geompos_in);
+	}
+
+	if (sf.metapos >= 0) {
+		char *meta = metabase + sf.metapos + meta_off[sf.segment];
+
+		for (size_t i = 0; i < sf.m; i++) {
+			long long k, v;
+			deserialize_long_long(&meta, &k);
+			deserialize_long_long(&meta, &v);
+			sf.keys.push_back(k);
+			sf.values.push_back(v);
+		}
+	} else {
+		for (size_t i = 0; i < sf.m; i++) {
+			long long k, v;
+			deserialize_long_long_io(geoms, &k, geompos_in);
+			deserialize_long_long_io(geoms, &v, geompos_in);
+			sf.keys.push_back(k);
+			sf.values.push_back(v);
+		}
+	}
+
+	deserialize_byte_io(geoms, &sf.feature_minzoom, geompos_in);
+
+	return sf;
+}

serial.hpp

@@ -44,6 +44,10 @@ struct serial_feature {
 	std::vector<long long> keys;
 	std::vector<long long> values;
 	long long metapos;
+
+	// XXX This isn't serialized. Should it be here?
+	long long bbox[4];
 };
 
 void serialize_feature(FILE *geomfile, serial_feature *sf, long long *geompos, const char *fname, long long wx, long long wy, bool include_minzoom);
+serial_feature deserialize_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);

tile.cpp

@@ -69,7 +69,6 @@ int metacmp(int m1, const std::vector<long long> &keys1, const std::vector<long
 int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2);
 
 struct coalesce {
-	char *meta;
 	char *stringpool;
 	std::vector<long long> keys;
 	std::vector<long long> values;
@@ -310,7 +309,6 @@ struct partial {
 	std::vector<long long> keys;
 	std::vector<long long> values;
 	std::vector<ssize_t> arc_polygon;
-	char *meta;
 	long long layer;
 	long long original_seq;
 	unsigned long long index;
@@ -1256,82 +1254,11 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 		}
 
 		while (1) {
-			signed char t;
+			serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y);
-			deserialize_byte_io(geoms, &t, geompos_in);
+			if (sf.t < 0) {
-			if (t < 0) {
 				break;
 			}
 
-			long long xlayer;
-			deserialize_long_long_io(geoms, &xlayer, geompos_in);
-
-			long long original_seq = 0;
-			if (xlayer & (1 << 5)) {
-				deserialize_long_long_io(geoms, &original_seq, geompos_in);
-			}
-
-			int tippecanoe_minzoom = -1, tippecanoe_maxzoom = -1;
-			unsigned long long id = 0;
-			bool has_id = false;
-			if (xlayer & (1 << 1)) {
-				deserialize_int_io(geoms, &tippecanoe_minzoom, geompos_in);
-			}
-			if (xlayer & (1 << 0)) {
-				deserialize_int_io(geoms, &tippecanoe_maxzoom, geompos_in);
-			}
-			if (xlayer & (1 << 2)) {
-				has_id = true;
-				deserialize_ulong_long_io(geoms, &id, geompos_in);
-			}
-			long long layer = xlayer >> 6;
-
-			int segment;
-			deserialize_int_io(geoms, &segment, geompos_in);
-
-			long long bbox[4];
-			unsigned long long index = 0;
-			long long extent = 0;
-
-			drawvec geom = decode_geometry(geoms, geompos_in, z, tx, ty, line_detail, bbox, initial_x[segment], initial_y[segment]);
-			if (xlayer & (1 << 4)) {
-				deserialize_ulong_long_io(geoms, &index, geompos_in);
-			}
-			if (xlayer & (1 << 3)) {
-				deserialize_long_long_io(geoms, &extent, geompos_in);
-			}
-
-			long long metastart = 0;
-			int m;
-			deserialize_int_io(geoms, &m, geompos_in);
-			if (m != 0) {
-				deserialize_long_long_io(geoms, &metastart, geompos_in);
-			}
-			char *meta = NULL;
-			std::vector<long long> metakeys, metavals;
-
-			if (metastart >= 0) {
-				meta = metabase + metastart + meta_off[segment];
-
-				for (int i = 0; i < m; i++) {
-					long long k, v;
-					deserialize_long_long(&meta, &k);
-					deserialize_long_long(&meta, &v);
-					metakeys.push_back(k);
-					metavals.push_back(v);
-				}
-			} else {
-				for (int i = 0; i < m; i++) {
-					long long k, v;
-					deserialize_long_long_io(geoms, &k, geompos_in);
-					deserialize_long_long_io(geoms, &v, geompos_in);
-					metakeys.push_back(k);
-					metavals.push_back(v);
-				}
-			}
-
-			signed char feature_minzoom;
-			deserialize_byte_io(geoms, &feature_minzoom, geompos_in);
-
 			double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10;
 			if (progress >= oprogress + 0.1) {
 				if (!quiet) {
@@ -1342,32 +1269,32 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 
 			original_features++;
 
-			int quick = quick_check(bbox, z, line_detail, buffer);
+			int quick = quick_check(sf.bbox, z, line_detail, buffer);
 			if (quick == 0) {
 				continue;
 			}
 
 			if (z == 0) {
-				if (bbox[0] < 0 || bbox[2] > 1LL << 32) {
+				if (sf.bbox[0] < 0 || sf.bbox[2] > 1LL << 32) {
 					// If the geometry extends off the edge of the world, concatenate on another copy
 					// shifted by 360 degrees, and then make sure both copies get clipped down to size.
 
-					size_t n = geom.size();
+					size_t n = sf.geometry.size();
 
-					if (bbox[0] < 0) {
+					if (sf.bbox[0] < 0) {
 						for (size_t i = 0; i < n; i++) {
-							geom.push_back(draw(geom[i].op, geom[i].x + (1LL << 32), geom[i].y));
+							sf.geometry.push_back(draw(sf.geometry[i].op, sf.geometry[i].x + (1LL << 32), sf.geometry[i].y));
 						}
 					}
 
-					if (bbox[2] > 1LL << 32) {
+					if (sf.bbox[2] > 1LL << 32) {
 						for (size_t i = 0; i < n; i++) {
-							geom.push_back(draw(geom[i].op, geom[i].x - (1LL << 32), geom[i].y));
+							sf.geometry.push_back(draw(sf.geometry[i].op, sf.geometry[i].x - (1LL << 32), sf.geometry[i].y));
 						}
 					}
 
-					bbox[0] = 0;
+					sf.bbox[0] = 0;
-					bbox[2] = 1LL << 32;
+					sf.bbox[2] = 1LL << 32;
 
 					quick = -1;
 				}
@@ -1382,70 +1309,70 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				// so that we can know whether the feature itself, or only the feature's
 				// bounding box, touches the tile.
 
-				if (t == VT_LINE) {
+				if (sf.t == VT_LINE) {
-					clipped = clip_lines(geom, z, line_detail, buffer);
+					clipped = clip_lines(sf.geometry, z, line_detail, buffer);
 				}
-				if (t == VT_POLYGON) {
+				if (sf.t == VT_POLYGON) {
-					clipped = simple_clip_poly(geom, z, line_detail, buffer);
+					clipped = simple_clip_poly(sf.geometry, z, line_detail, buffer);
 				}
-				if (t == VT_POINT) {
+				if (sf.t == VT_POINT) {
-					clipped = clip_point(geom, z, line_detail, buffer);
+					clipped = clip_point(sf.geometry, z, line_detail, buffer);
 				}
 
-				clipped = remove_noop(clipped, t, 0);
+				clipped = remove_noop(clipped, sf.t, 0);
 
 				// Must clip at z0 even if we don't want clipping, to handle features
 				// that are duplicated across the date line
 
 				if (prevent[P_DUPLICATION] && z != 0) {
-					if (point_within_tile((bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2, z, line_detail, buffer)) {
+					if (point_within_tile((sf.bbox[0] + sf.bbox[2]) / 2, (sf.bbox[1] + sf.bbox[3]) / 2, z, line_detail, buffer)) {
-						// geom is unchanged
+						// sf.geometry is unchanged
 					} else {
-						geom.clear();
+						sf.geometry.clear();
 					}
 				} else if (prevent[P_CLIPPING] && z != 0) {
 					if (clipped.size() == 0) {
-						geom.clear();
+						sf.geometry.clear();
 					} else {
-						// geom is unchanged
+						// sf.geometry is unchanged
 					}
 				} else {
-					geom = clipped;
+					sf.geometry = clipped;
 				}
 			}
 
-			if (geom.size() > 0) {
+			if (sf.geometry.size() > 0) {
 				unclipped_features++;
 			}
 
 			if (first_time && pass == 1) { /* only write out the next zoom once, even if we retry */
-				rewrite(geom, z, nextzoom, maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, child_shards, max_zoom_increment, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom, segment, initial_x, initial_y, m, metakeys, metavals, has_id, id, index, extent);
+				rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.m, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent);
 			}
 
 			if (z < minzoom) {
 				continue;
 			}
 
-			if (tippecanoe_minzoom != -1 && z < tippecanoe_minzoom) {
+			if (sf.tippecanoe_minzoom != -1 && z < sf.tippecanoe_minzoom) {
 				continue;
 			}
-			if (tippecanoe_maxzoom != -1 && z > tippecanoe_maxzoom) {
+			if (sf.tippecanoe_maxzoom != -1 && z > sf.tippecanoe_maxzoom) {
 				continue;
 			}
-			if (tippecanoe_minzoom == -1 && z < feature_minzoom) {
+			if (sf.tippecanoe_minzoom == -1 && z < sf.feature_minzoom) {
 				continue;
 			}
 
 			if (prefilter != NULL) {
 				mvt_layer tmp_layer;
 				tmp_layer.extent = 1LL << 32;
-				tmp_layer.name = (*layer_unmaps)[segment][layer];
+				tmp_layer.name = (*layer_unmaps)[sf.segment][sf.layer];
 
 				mvt_feature tmp_feature;
-				tmp_feature.type = t;
+				tmp_feature.type = sf.t;
-				tmp_feature.geometry = to_feature(geom);
+				tmp_feature.geometry = to_feature(sf.geometry);
-				tmp_feature.id = id;
+				tmp_feature.id = sf.id;
-				tmp_feature.has_id = id;
+				tmp_feature.has_id = sf.has_id;
 
 				// Offset from tile coordinates back to world coordinates
 				unsigned sx = 0, sy = 0;
@@ -1458,27 +1385,27 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 					tmp_feature.geometry[i].y += sy;
 				}
 
-				decode_meta(m, metakeys, metavals, stringpool + pool_off[segment], tmp_layer, tmp_feature);
+				decode_meta(sf.m, sf.keys, sf.values, stringpool + pool_off[sf.segment], tmp_layer, tmp_feature);
 				tmp_layer.features.push_back(tmp_feature);
 
 				layer_to_geojson(prefilter_fd, tmp_layer, 0, 0, 0, false, true);
 			}
 
 			if (gamma > 0) {
-				if (manage_gap(index, &previndex, scale, gamma, &gap)) {
+				if (manage_gap(sf.index, &previndex, scale, gamma, &gap)) {
 					continue;
 				}
 			}
 
 			if (additional[A_DROP_DENSEST_AS_NEEDED]) {
-				indices.push_back(index);
+				indices.push_back(sf.index);
-				if (index - merge_previndex < mingap) {
+				if (sf.index - merge_previndex < mingap) {
 					continue;
 				}
 			}
 			if (additional[A_DROP_SMALLEST_AS_NEEDED]) {
-				extents.push_back(extent);
+				extents.push_back(sf.extent);
-				if (extent <= minextent && t != VT_POINT) {
+				if (sf.extent <= minextent && sf.t != VT_POINT) {
 					continue;
 				}
 			}
@@ -1490,8 +1417,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				// that standard, so that duplicates aren't reported as infinitely dense.
 
 				double o_density_previndex = density_previndex;
-				if (!manage_gap(index, &density_previndex, scale, 1, &density_gap)) {
+				if (!manage_gap(sf.index, &density_previndex, scale, 1, &density_gap)) {
-					spacing = (index - o_density_previndex) / scale;
+					spacing = (sf.index - o_density_previndex) / scale;
 				}
 			}
 
@@ -1502,39 +1429,38 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 			fraction_accum -= 1;
 
 			bool reduced = false;
-			if (t == VT_POLYGON) {
+			if (sf.t == VT_POLYGON) {
 				if (!prevent[P_TINY_POLYGON_REDUCTION] && !additional[A_GRID_LOW_ZOOMS]) {
-					geom = reduce_tiny_poly(geom, z, line_detail, &reduced, &accum_area);
+					sf.geometry = reduce_tiny_poly(sf.geometry, z, line_detail, &reduced, &accum_area);
 				}
 				has_polygons = true;
 			}
 
-			if (geom.size() > 0) {
+			if (sf.geometry.size() > 0) {
 				partial p;
-				p.geoms.push_back(geom);
+				p.geoms.push_back(sf.geometry);
-				p.layer = layer;
+				p.layer = sf.layer;
-				p.m = m;
+				p.m = sf.m;
-				p.meta = meta;
+				p.t = sf.t;
-				p.t = t;
+				p.segment = sf.segment;
-				p.segment = segment;
+				p.original_seq = sf.seq;
-				p.original_seq = original_seq;
 				p.reduced = reduced;
 				p.z = z;
 				p.line_detail = line_detail;
 				p.maxzoom = maxzoom;
-				p.keys = metakeys;
+				p.keys = sf.keys;
-				p.values = metavals;
+				p.values = sf.values;
 				p.spacing = spacing;
 				p.simplification = simplification;
-				p.id = id;
+				p.id = sf.id;
-				p.has_id = has_id;
+				p.has_id = sf.has_id;
 				p.index2 = merge_previndex;
-				p.index = index;
+				p.index = sf.index;
 				p.renamed = -1;
 				partials.push_back(p);
 			}
 
-			merge_previndex = index;
+			merge_previndex = sf.index;
 		}
 
 		if (prefilter != NULL) {
@@ -1616,7 +1542,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 					c.coalesced = false;
 					c.original_seq = original_seq;
 					c.m = partials[i].m;
-					c.meta = partials[i].meta;
 					c.stringpool = stringpool + pool_off[partials[i].segment];
 					c.keys = partials[i].keys;
 					c.values = partials[i].values;