Stick a hash search on the front of the tree search in addpool

pool.cpp

@@ -7,32 +7,33 @@
 #include "memfile.hpp"
 #include "pool.hpp"
 #include "errors.hpp"
+#include "text.hpp"
 
 int swizzlecmp(const char *a, const char *b) {
-	ssize_t alen = strlen(a);
+	unsigned long long h1 = fnv1a(a, 0);
-	ssize_t blen = strlen(b);
+	unsigned long long h2 = fnv1a(b, 0);
 
-	if (strcmp(a, b) == 0) {
-		return 0;
-	}
-
-	long long hash1 = 0, hash2 = 0;
-	for (ssize_t i = alen - 1; i >= 0; i--) {
-		hash1 = (hash1 * 37 + a[i]) & INT_MAX;
-	}
-	for (ssize_t i = blen - 1; i >= 0; i--) {
-		hash2 = (hash2 * 37 + b[i]) & INT_MAX;
-	}
-
-	int h1 = hash1, h2 = hash2;
 	if (h1 == h2) {
 		return strcmp(a, b);
-	}
+	} else {
-
 		return h1 - h2;
+	}
 }
 
-long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type) {
+long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type, std::vector<ssize_t> &dedup) {
+	unsigned long long hash = fnv1a(s, type);
+	size_t hash_off = hash % dedup.size();
+
+	if (dedup[hash_off] >= 0 &&
+	    dedup[hash_off] + 1 < (ssize_t) poolfile->map.size() &&
+	    poolfile->map[dedup[hash_off]] == type &&
+	    strcmp(poolfile->map.c_str() + dedup[hash_off] + 1, s) == 0) {
+		// printf("hit for %s\n", s);
+		return dedup[hash_off];
+	} else {
+		// printf("miss for %s\n", s);
+	}
+
 	unsigned long *sp = &treefile->tree;
 	size_t depth = 0;
 
@@ -54,6 +55,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
 		} else if (cmp > 0) {
 			sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->right);
 		} else {
+			dedup[hash_off] = ((struct stringpool *) (treefile->map.c_str() + *sp))->off;
 			return ((struct stringpool *) (treefile->map.c_str() + *sp))->off;
 		}
 

pool.hpp

@@ -7,6 +7,6 @@ struct stringpool {
 	unsigned long off = 0;
 };
 
-long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type);
+long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type, std::vector<ssize_t> &dedup);
 
 #endif

serial.cpp

@@ -837,8 +837,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
 	}
 
 	for (size_t i = 0; i < sf.full_keys.size(); i++) {
-		sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string));
+		sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string, r->key_dedup));
-		sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type));
+		sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type, r->value_dedup));
 	}
 
 	long long geomstart = r->geompos;

serial.hpp

@@ -101,14 +101,15 @@ struct reader {
 	std::atomic<long long> nodepos;
 
 	long long file_bbox[4] = {0, 0, 0, 0};
-
 	long long file_bbox1[4] = {0xFFFFFFFF, 0xFFFFFFFF, 0, 0};	      // standard -180 to 180 world plane
 	long long file_bbox2[4] = {0x1FFFFFFFF, 0xFFFFFFFF, 0x100000000, 0};  // 0 to 360 world plane
 
 	struct stat geomst {};
-
 	char *geom_map = NULL;
 
+	std::vector<ssize_t> key_dedup = std::vector<ssize_t>(655536, -1);
+	std::vector<ssize_t> value_dedup = std::vector<ssize_t>(655536, -1);
+
 	reader()
 	    : geompos(0), indexpos(0), vertexpos(0), nodepos(0) {
 	}