Reduce memory use of polygon shard detection (#139)

* Crunch down memory required per polygon joint

* *Actually* reduce the size of the structure

* Update version and changelog

CHANGELOG.md

@@ -1,3 +1,11 @@
+# 2.32.1
+
+* Reduce memory usage of --no-simplification-of-shared-nodes for polygons
+
+# 2.32.0
+
+* Extend --no-simplification-of-shared-nodes to also simplify shared polygon borders consistently
+
 # 2.31.0
 
 * Fix tile-join crash when trying to join empty tilesets

mbtiles.cpp

@@ -99,17 +99,21 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) {
 	return outdb;
 }
 
-void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size) {
+unsigned long long fnv1a(std::string const &s) {
 	// Store tiles by a hash of their contents (fnv1a 64-bit)
 	// http://www.isthe.com/chongo/tech/comp/fnv/
 	const unsigned long long fnv_offset_basis = 14695981039346656037u;
 	const unsigned long long fnv_prime = 1099511628211u;
 	unsigned long long h = fnv_offset_basis;
-	for (int i = 0; i < size; i++) {
+	for (size_t i = 0; i < s.size(); i++) {
-		h ^= (unsigned char) data[i];
+		h ^= (unsigned char) s[i];
 		h *= fnv_prime;
 	}
-	std::string hash = std::to_string(h);
+	return h;
+}
+
+void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size) {
+	std::string hash = std::to_string(fnv1a(std::string(data, size)));
 
 	// following https://github.com/mapbox/node-mbtiles/blob/master/lib/mbtiles.js
 

mbtiles.hpp

@@ -85,4 +85,6 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
 
 void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &layername, type_and_string const &val);
 
+unsigned long long fnv1a(std::string const &s);
+
 #endif

tile.cpp

@@ -1295,26 +1295,33 @@ long long choose_minextent(std::vector<long long> &extents, double f) {
 }
 
 struct joint {
-	draw p1;
+	long long x : 34;  // enough to wrap around the world either way
-	draw mid;
+	unsigned long long p1 : 64 - 34;
-	draw p2;
+
+	long long y : 33;  // enough to touch the top and bottom of the world
+	unsigned long long p2 : 64 - 34;
 
 	joint(draw one, draw hinge, draw two) {
 		if (one < two) {
-			p1 = one;
+			std::swap(one, two);
-			p2 = two;
-		} else {
-			p1 = two;
-			p2 = one;
 		}
 
-		mid = hinge;
+		long long coord1[2] = {one.x, one.y};
+		long long coord2[2] = {two.x, two.y};
+		p1 = fnv1a(std::string((const char *) &coord1, sizeof(coord1)));
+		p2 = fnv1a(std::string((const char *) &coord2, sizeof(coord2)));
+
+		x = hinge.x;
+		y = hinge.y;
 	}
 
 	bool operator<(const joint &o) const {
-		if (mid < o.mid) {
+		if (y < o.y) {
 			return true;
-		} else if (mid == o.mid) {
+		} else if (y == o.y) {
+			if (x < o.x) {
+				return true;
+			} else if (x == o.x) {
 				if (p1 < o.p1) {
 					return true;
 				} else if (p1 == o.p1) {
@@ -1323,6 +1330,7 @@ struct joint {
 					}
 				}
 			}
+		}
 
 		return false;
 	}
@@ -2433,10 +2441,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 
 			std::sort(shared_joints.begin(), shared_joints.end());
 			for (size_t i = 0; i + 1 < shared_joints.size(); i++) {
-				if (shared_joints[i].mid == shared_joints[i + 1].mid) {
+				if (shared_joints[i].x == shared_joints[i + 1].x &&
+				    shared_joints[i].y == shared_joints[i + 1].y) {
 					if (shared_joints[i].p1 != shared_joints[i + 1].p1 ||
 					    shared_joints[i].p2 != shared_joints[i + 1].p2) {
-						shared_nodes.push_back(shared_joints[i].mid);
+						shared_nodes.push_back(draw(VT_MOVETO, shared_joints[i].x, shared_joints[i].y));
 					}
 				}
 			}

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.31.0"
+#define VERSION "v2.32.1"
 
 #endif