Merge pull request #247 from mapbox/less-memory

Pack structures tighter to use a little less memory

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.11.6
+
+* Reduce the size of critical data structures to reduce dynamic memory use
+
 ## 1.11.5
 
 * Let zoom level 0 have just as much extent and buffer as any other zoom

geometry.cpp

@@ -767,7 +767,7 @@ drawvec simple_clip_poly(drawvec &geom, long long minx, long long miny, long lon
 }
 
 drawvec simple_clip_poly(drawvec &geom, int z, int detail, int buffer) {
-	long long area = area = 1LL << (32 - z);
+	long long area = 1LL << (32 - z);
 	long long clip_buffer = buffer * area / 256;
 
 	return simple_clip_poly(geom, -clip_buffer, -clip_buffer, area + clip_buffer, area + clip_buffer);

geometry.hpp

@@ -11,11 +11,14 @@
 #define VT_NUMBER 2
 #define VT_BOOLEAN 7
 
+// The bitfield is to make sizeof(draw) be 16 instead of 24
+// at the cost, apparently, of a 0.7% increase in running time
+// for packing and unpacking.
 struct draw {
+	long long x : 40;
 	signed char op;
-	long long x;
+	long long y : 40;
-	long long y;
+	signed char necessary;
-	int necessary;
 
 	draw(int nop, long long nx, long long ny) {
 		this->op = nop;

mvt.hpp

@@ -8,9 +8,9 @@ enum mvt_operation {
 };
 
 struct mvt_geometry {
+	int x;
+	int y;
 	int /* mvt_operation */ op;
-	long long x;
-	long long y;
 
 	mvt_geometry(int op, long long x, long long y);
 };
@@ -23,8 +23,8 @@ enum mvt_geometry_type {
 
 struct mvt_feature {
 	std::vector<unsigned> tags;
-	int /* mvt_geometry_type */ type;
 	std::vector<mvt_geometry> geometry;
+	int /* mvt_geometry_type */ type;
 };
 
 enum mvt_value_type {

tile.cpp

@@ -62,15 +62,15 @@ int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2);
 static int is_integer(const char *s, long long *v);
 
 struct coalesce {
-	int type;
-	drawvec geom;
-	int m;
 	char *meta;
 	char *stringpool;
+	drawvec geom;
 	unsigned long long index;
 	unsigned long long index2;
-	bool coalesced;
 	long long original_seq;
+	int type;
+	int m;
+	bool coalesced;
 
 	bool operator<(const coalesce &o) const {
 		int cmp = coalindexcmp(this, &o);
@@ -372,20 +372,20 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
 
 struct partial {
 	std::vector<drawvec> geoms;
-	long long layer;
-	int m;
 	char *meta;
-	signed char t;
-	int segment;
-	long long original_seq;
-	bool reduced;
-	unsigned long long index;
-	unsigned long long index2;
-	int z;
-	int line_detail;
 	int *prevent;
 	int *additional;
+	long long layer;
+	long long original_seq;
+	unsigned long long index;
+	unsigned long long index2;
+	int m;
+	int segment;
+	bool reduced;
+	int z;
+	int line_detail;
 	int maxzoom;
+	signed char t;
 };
 
 struct partial_arg {
@@ -855,8 +855,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 		}
 
 		for (size_t i = 0; i < partials.size(); i++) {
-			std::vector<drawvec> pgeoms = partials[i].geoms;
+			std::vector<drawvec> &pgeoms = partials[i].geoms;
-			partials[i].geoms.clear();  // avoid keeping two copies in memory
 			long long layer = partials[i].layer;
 			signed char t = partials[i].t;
 			long long original_seq = partials[i].original_seq;
@@ -871,6 +870,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 					c.index = partials[i].index;
 					c.index2 = partials[i].index2;
 					c.geom = pgeoms[j];
+					pgeoms[j].clear();
 					c.coalesced = false;
 					c.original_seq = original_seq;
 					c.m = partials[i].m;
@@ -882,6 +882,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 			}
 		}
 
+		partials.clear();
+
 		int j;
 		for (j = 0; j < child_shards; j++) {
 			if (within[j]) {
@@ -966,6 +968,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
 				feature.type = features[k][x].type;
 				feature.geometry = to_feature(features[k][x].geom);
 				count += features[k][x].geom.size();
+				features[k][x].geom.clear();
 
 				decode_meta(features[k][x].m, &features[k][x].meta, features[k][x].stringpool, layer, feature);
 				layer.features.push_back(feature);

version.hpp

@@ -1 +1 @@
-#define VERSION "tippecanoe v1.11.5\n"
+#define VERSION "tippecanoe v1.11.6\n"