From 711a75789518853c3e33148ab7314d5817c04d0a Mon Sep 17 00:00:00 2001
From: Erica Fischer <enf@felt.com>
Date: Fri, 2 Feb 2024 13:28:39 -0800
Subject: [PATCH] Stick a hash search on the front of the tree search in
 addpool

---
 pool.cpp   | 38 ++++++++++++++++++++------------------
 pool.hpp   |  2 +-
 serial.cpp |  4 ++--
 serial.hpp |  5 +++--
 4 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/pool.cpp b/pool.cpp
index 17e37c57..744cda50 100644
--- a/pool.cpp
+++ b/pool.cpp
@@ -7,32 +7,33 @@
 #include "memfile.hpp"
 #include "pool.hpp"
 #include "errors.hpp"
+#include "text.hpp"
 
 int swizzlecmp(const char *a, const char *b) {
-	ssize_t alen = strlen(a);
-	ssize_t blen = strlen(b);
+	unsigned long long h1 = fnv1a(a, 0);
+	unsigned long long h2 = fnv1a(b, 0);
 
-	if (strcmp(a, b) == 0) {
-		return 0;
-	}
-
-	long long hash1 = 0, hash2 = 0;
-	for (ssize_t i = alen - 1; i >= 0; i--) {
-		hash1 = (hash1 * 37 + a[i]) & INT_MAX;
-	}
-	for (ssize_t i = blen - 1; i >= 0; i--) {
-		hash2 = (hash2 * 37 + b[i]) & INT_MAX;
-	}
-
-	int h1 = hash1, h2 = hash2;
 	if (h1 == h2) {
 		return strcmp(a, b);
+	} else {
+		return h1 - h2;
 	}
-
-	return h1 - h2;
 }
 
-long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type) {
+long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type, std::vector<ssize_t> &dedup) {
+	unsigned long long hash = fnv1a(s, type);
+	size_t hash_off = hash % dedup.size();
+
+	if (dedup[hash_off] >= 0 &&
+	    dedup[hash_off] + 1 < (ssize_t) poolfile->map.size() &&
+	    poolfile->map[dedup[hash_off]] == type &&
+	    strcmp(poolfile->map.c_str() + dedup[hash_off] + 1, s) == 0) {
+		// printf("hit for %s\n", s);
+		return dedup[hash_off];
+	} else {
+		// printf("miss for %s\n", s);
+	}
+
 	unsigned long *sp = &treefile->tree;
 	size_t depth = 0;
 
@@ -54,6 +55,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
 		} else if (cmp > 0) {
 			sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->right);
 		} else {
+			dedup[hash_off] = ((struct stringpool *) (treefile->map.c_str() + *sp))->off;
 			return ((struct stringpool *) (treefile->map.c_str() + *sp))->off;
 		}
 
diff --git a/pool.hpp b/pool.hpp
index 54ac4ea0..fdf631c5 100644
--- a/pool.hpp
+++ b/pool.hpp
@@ -7,6 +7,6 @@ struct stringpool {
 	unsigned long off = 0;
 };
 
-long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type);
+long long addpool(struct memfile *poolfile, struct memfile *treefile, const char *s, char type, std::vector<ssize_t> &dedup);
 
 #endif
diff --git a/serial.cpp b/serial.cpp
index 67f926c4..a33ae976 100644
--- a/serial.cpp
+++ b/serial.cpp
@@ -837,8 +837,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
 	}
 
 	for (size_t i = 0; i < sf.full_keys.size(); i++) {
-		sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string));
-		sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type));
+		sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string, r->key_dedup));
+		sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type, r->value_dedup));
 	}
 
 	long long geomstart = r->geompos;
diff --git a/serial.hpp b/serial.hpp
index fecbdf41..f4ad239f 100644
--- a/serial.hpp
+++ b/serial.hpp
@@ -101,14 +101,15 @@ struct reader {
 	std::atomic<long long> nodepos;
 
 	long long file_bbox[4] = {0, 0, 0, 0};
-
 	long long file_bbox1[4] = {0xFFFFFFFF, 0xFFFFFFFF, 0, 0};	      // standard -180 to 180 world plane
 	long long file_bbox2[4] = {0x1FFFFFFFF, 0xFFFFFFFF, 0x100000000, 0};  // 0 to 360 world plane
 
 	struct stat geomst {};
-
 	char *geom_map = NULL;
 
+	std::vector<ssize_t> key_dedup = std::vector<ssize_t>(655536, -1);
+	std::vector<ssize_t> value_dedup = std::vector<ssize_t>(655536, -1);
+
 	reader()
 	    : geompos(0), indexpos(0), vertexpos(0), nodepos(0) {
 	}