Reduce memory consumption during tiling (#177)

* Drop duplicate geometries sooner in coalescing-as-needed

* Concatenate geometries before partial cleaning

* Spend less time with two feature representations in memory

* Further reduce in-memory duplication

* Remove another copy

* Don't keep duplicates in memory while coalescing

* Also clear out the mvt layer once it is no longer needed

* Update changelog

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.39.0
+
+* Reduce memory usage during tiling
+
 # 2.38.0
 
 * Tolerate polygon rings with insuffiently many points in input

tile.cpp

@@ -1898,6 +1898,19 @@ void add_sample_to(std::vector<T> &vals, T val, size_t &increment, size_t seq) {
 	}
 }
 
+void coalesce_geometry(partial &p, serial_feature &sf) {
+	// if the geometry being coalesced on is an exact duplicate
+	// of an existing geometry, just drop it
+
+	for (size_t i = 0; i < p.geoms.size(); i++) {
+		if (p.geoms[i] == sf.geometry) {
+			return;
+		}
+	}
+
+	p.geoms.push_back(sf.geometry);
+}
+
 long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy, bool compressed_input, struct node *shared_nodes_map, size_t nodepos) {
 	double merge_fraction = 1;
 	double mingap_fraction = 1;
@@ -2145,7 +2158,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					indices.push_back(sf.index);
 				}
 				if (sf.index - merge_previndex < mingap && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
-					partials[which_partial].geoms.push_back(sf.geometry);
+					coalesce_geometry(partials[which_partial], sf);
 					partials[which_partial].coalesced = true;
 					coalesced_area += sf.extent;
 					preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
@@ -2164,7 +2177,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			} else if (additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
 				add_sample_to(extents, sf.extent, extents_increment, seq);
 				if (minextent != 0 && sf.extent + coalesced_area <= minextent && find_partial(partials, sf, which_partial, layer_unmaps, minextent)) {
-					partials[which_partial].geoms.push_back(sf.geometry);
+					coalesce_geometry(partials[which_partial], sf);
 					partials[which_partial].coalesced = true;
 					coalesced_area += sf.extent;
 					preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
@@ -2188,7 +2201,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			fraction_accum += fraction;
 			if (fraction_accum < 1 && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
 				if (additional[A_COALESCE_FRACTION_AS_NEEDED]) {
-					partials[which_partial].geoms.push_back(sf.geometry);
+					coalesce_geometry(partials[which_partial], sf);
 					partials[which_partial].coalesced = true;
 					coalesced_area += sf.extent;
 					strategy->coalesced_as_needed++;
@@ -2290,12 +2303,20 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 						for (; simplified_geometry_through < partials.size(); simplified_geometry_through++) {
 							simplify_partial(&partials[simplified_geometry_through], dv, shared_nodes_map, nodepos);
 
-							for (auto &g : partials[simplified_geometry_through].geoms) {
 							if (partials[simplified_geometry_through].t == VT_POLYGON) {
-									// don't scale up because this is still world coordinates
+								drawvec to_clean;
-									g = clean_or_clip_poly(g, 0, 0, false, false);
+
+								for (auto &g : partials[simplified_geometry_through].geoms) {
+									for (auto &d : g) {
+										to_clean.push_back(d);
 									}
 								}
+
+								// don't scale up because this is still world coordinates
+								to_clean = clean_or_clip_poly(to_clean, 0, 0, false, false);
+								partials[simplified_geometry_through].geoms.clear();
+								partials[simplified_geometry_through].geoms.push_back(to_clean);
+							}
 						}
 
 						unsimplified_geometry_size = 0;
@@ -2430,7 +2451,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			}
 		}
 
-		for (size_t i = 0; i < partials.size(); i++) {
+		std::reverse(partials.begin(), partials.end());
+		for (ssize_t i = partials.size() - 1; i >= 0; i--) {
 			std::vector<drawvec> &pgeoms = partials[i].geoms;
 			signed char t = partials[i].t;
 			long long original_seq = partials[i].original_seq;
@@ -2474,6 +2496,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					l->second.push_back(c);
 				}
 			}
+
+			partials.erase(partials.begin() + i);
 		}
 
 		partials.clear();
@@ -2494,32 +2518,38 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				std::sort(layer_features.begin(), layer_features.end());
 			}
 
-			std::vector<coalesce> out;
+			if (additional[A_COALESCE]) {
+				// coalesce adjacent identical features if requested
+
+				size_t out = 0;
 				if (layer_features.size() > 0) {
-				out.push_back(layer_features[0]);
+					out++;
 				}
+
 				for (size_t x = 1; x < layer_features.size(); x++) {
-				size_t y = out.size() - 1;
+					size_t y = out - 1;
 
-#if 0
+					if (out > 0 && coalcmp(&layer_features[x], &layer_features[y]) == 0) {
-				if (out.size() > 0 && coalcmp(&layer_features[x], &out[y]) < 0) {
-					fprintf(stderr, "\nfeature out of order\n");
-				}
-#endif
-
-				if (additional[A_COALESCE] && out.size() > 0 && coalcmp(&layer_features[x], &out[y]) == 0) {
 						for (size_t g = 0; g < layer_features[x].geom.size(); g++) {
-						out[y].geom.push_back(layer_features[x].geom[g]);
+							layer_features[y].geom.push_back(layer_features[x].geom[g]);
 						}
-					out[y].coalesced = true;
+						layer_features[y].coalesced = true;
 					} else {
-					out.push_back(layer_features[x]);
+						layer_features[out++] = layer_features[x];
 					}
 				}
 
-			layer_features = out;
+				layer_features.resize(out);
+			}
+
+			{
+				// clean up coalesced linestrings by simplification
+				// and coalesced polygons by cleaning
+				//
+				// then close polygons
+
+				size_t out = 0;
 
-			out.clear();
 				for (size_t x = 0; x < layer_features.size(); x++) {
 					if (layer_features[x].coalesced && layer_features[x].type == VT_LINE) {
 						layer_features[x].geom = remove_noop(layer_features[x].geom, layer_features[x].type, 0);
@@ -2540,10 +2570,12 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					}
 
 					if (layer_features[x].geom.size() > 0) {
-					out.push_back(layer_features[x]);
+						layer_features[out++] = layer_features[x];
 					}
 				}
-			layer_features = out;
+
+				layer_features.resize(out);
+			}
 
 			if (prevent[P_INPUT_ORDER]) {
 				std::sort(layer_features.begin(), layer_features.end(), preservecmp);
@@ -2565,16 +2597,19 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 		}
 
 		mvt_tile tile;
+		size_t totalsize = 0;
 
 		for (auto layer_iterator = layers.begin(); layer_iterator != layers.end(); ++layer_iterator) {
 			std::vector<coalesce> &layer_features = layer_iterator->second;
+			totalsize += layer_features.size();
 
 			mvt_layer layer;
 			layer.name = layer_iterator->first;
 			layer.version = 2;
 			layer.extent = 1 << tile_detail;
 
-			for (size_t x = 0; x < layer_features.size(); x++) {
+			std::reverse(layer_features.begin(), layer_features.end());
+			for (ssize_t x = layer_features.size() - 1; x >= 0; x--) {
 				mvt_feature feature;
 
 				if (layer_features[x].type == VT_LINE || layer_features[x].type == VT_POLYGON) {
@@ -2622,6 +2657,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				}
 
 				layer.features.push_back(feature);
+				layer_features.erase(layer_features.begin() + x);
 			}
 
 			if (layer.features.size() > 0) {
@@ -2638,12 +2674,6 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			fprintf(stderr, "Is your data in the wrong projection? It should be in WGS84/EPSG:4326.\n");
 		}
 
-		size_t totalsize = 0;
-		for (auto layer_iterator = layers.begin(); layer_iterator != layers.end(); ++layer_iterator) {
-			std::vector<coalesce> &layer_features = layer_iterator->second;
-			totalsize += layer_features.size();
-		}
-
 		size_t passes = pass + 1;
 		double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + pass) / passes + z) / (maxzoom + 1) * 1000) / 10;
 		if (progress >= oprogress + 0.1) {
@@ -2744,6 +2774,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			std::string compressed;
 			std::string pbf = tile.encode();
 
+			tile.layers.clear();
+
 			if (!prevent[P_TILE_COMPRESSION]) {
 				compress(pbf, compressed, true);
 			} else {

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.38.0"
+#define VERSION "v2.39.0"
 
 #endif