Externalizing polygon shard detection (#146)

* Start of externalizing polygon shard detection

* Completely untested external quicksort

* Add unit test for external quicksort

* Remember to clean up temporary files

* Sort and scan the vertices

* Bring over more vertex logic

* Make nodes from vertices

* Checkpoint on switching over to global shared nodes

* Do the thing

* Revert unintended change to coalesced linestring behavior

* Take shared nodes into account in early simplification

* Let it do more sorting in memory

* Fix overnoding of collinear linestrings

* Remove duplicate nodes, since only duplicate vertices now matter

* Remember to delete temporary files

* Fix out of bounds memory access below, apparently

* Fix the actual undefined behavior

* Still running out of memory in one case. Find out where.

* Forgot the conditional

* Try again to make it not run out of memory

* Update version and changelog

serial.hpp

@@ -78,14 +78,20 @@ struct reader {
 	int treefd = -1;
 	int geomfd = -1;
 	int indexfd = -1;
+	int vertexfd = -1;
+	int nodefd = -1;
 
 	struct memfile *poolfile = NULL;
 	struct memfile *treefile = NULL;
 	FILE *geomfile = NULL;
 	FILE *indexfile = NULL;
+	FILE *vertexfile = NULL;
+	FILE *nodefile = NULL;
 
 	std::atomic<long long> geompos;
 	std::atomic<long long> indexpos;
+	std::atomic<long long> vertexpos;
+	std::atomic<long long> nodepos;
 
 	long long file_bbox[4] = {0, 0, 0, 0};
 
@@ -97,7 +103,7 @@ struct reader {
 	char *geom_map = NULL;
 
 	reader()
-	    : geompos(0), indexpos(0) {
+	    : geompos(0), indexpos(0), vertexpos(0), nodepos(0) {
 	}
 
 	reader(reader const &r) {
@@ -105,11 +111,15 @@ struct reader {
 		treefd = r.treefd;
 		geomfd = r.geomfd;
 		indexfd = r.indexfd;
+		vertexfd = r.vertexfd;
+		nodefd = r.nodefd;
 
 		poolfile = r.poolfile;
 		treefile = r.treefile;
 		geomfile = r.geomfile;
 		indexfile = r.indexfile;
+		vertexfile = r.vertexfile;
+		nodefile = r.nodefile;
 
 		long long p = r.geompos;
 		geompos = p;
@@ -117,6 +127,12 @@ struct reader {
 		p = r.indexpos;
 		indexpos = p;
 
+		p = r.vertexpos;
+		vertexpos = p;
+
+		p = r.nodepos;
+		nodepos = p;
+
 		memcpy(file_bbox, r.file_bbox, sizeof(file_bbox));
 
 		geomst = r.geomst;
@@ -158,6 +174,52 @@ struct serialization_state {
 	int exclude_all = 0;
 };
 
+struct vertex {
+	// these are scaled geometry,
+	// but because scaling is disabled if P_SHARED_NODES is set,
+	// they are effectively also world coordinates
+	draw p1;
+	draw mid;
+	draw p2;
+
+	vertex(draw one, draw joint, draw two) {
+		if (one < two) {
+			p1 = one;
+			p2 = two;
+		} else {
+			p1 = two;
+			p2 = one;
+		}
+
+		mid = joint;
+	}
+
+	bool operator<(const vertex &v) const {
+		if (mid < v.mid) {
+			return true;
+		} else if (mid == v.mid) {
+			if (p1 < v.p1) {
+				return true;
+			} else if (p1 == v.p1) {
+				if (p2 < v.p2) {
+					return true;
+				}
+			}
+		}
+
+		return false;
+	}
+};
+
+struct node {
+	// this is in quadkey coordinates so that the nodes for each tile
+	// will be adjacent in memory, reducing potential thrashing during
+	// the binary search.
+	unsigned long long index;
+};
+
+int nodecmp(const void *void1, const void *void2);
+
 int serialize_feature(struct serialization_state *sst, serial_feature &sf);
 void coerce_value(std::string const &key, int &vt, std::string &val, std::map<std::string, int> const *attribute_types);