From 5c681e50d9963a3219fc3d3a475abc4a3aaa8537 Mon Sep 17 00:00:00 2001
From: Erica Fischer <enf@felt.com>
Date: Fri, 10 Feb 2023 12:21:20 -0800
Subject: [PATCH] Weight outlieriness by categoricality

---
 mbtiles.cpp | 24 +++++++++++++++---------
 mbtiles.hpp |  1 +
 tile.cpp    | 36 +++++++++++++++++++++++-------------
 3 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/mbtiles.cpp b/mbtiles.cpp
index 8c445b95..693aea02 100644
--- a/mbtiles.cpp
+++ b/mbtiles.cpp
@@ -908,6 +908,20 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
 	return out;
 }
 
+double categorical_hash(std::string const &val) {
+	unsigned hash = 0;
+	for (size_t i = 0; i < val.size(); i++) {
+		// https://en.wikipedia.org/wiki/Hash_function#Fibonacci_hashing
+		hash = hash * 2654435769 + val[i];
+	}
+	// extra multiply so even single-byte values are distributed
+	// around the circle rather than clumped together on one side.
+	hash = hash * 2654435769;
+	double angle = ((double) hash) / UINT_MAX * 2 * M_PI;
+
+	return angle;
+}
+
 void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &attrib, type_and_string const &val) {
 	if (val.type == mvt_null) {
 		return;
@@ -944,15 +958,7 @@ void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, s
 		}
 	}
 
-	unsigned hash = 0;
-	for (size_t i = 0; i < val.string.size(); i++) {
-		// https://en.wikipedia.org/wiki/Hash_function#Fibonacci_hashing
-		hash = hash * 2654435769 + val.string[i];
-	}
-	// extra multiply so even single-byte values are distributed
-	// around the circle rather than clumped together on one side.
-	hash = hash * 2654435769;
-	double angle = ((double) hash) / UINT_MAX * 2 * M_PI;
+	double angle = categorical_hash(val.string);
 	double x = cos(angle);
 	double y = sin(angle);
 
diff --git a/mbtiles.hpp b/mbtiles.hpp
index 9dcae9b0..393ade9d 100644
--- a/mbtiles.hpp
+++ b/mbtiles.hpp
@@ -93,5 +93,6 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
 std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps, bool trunc);
 
 void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &layername, type_and_string const &val);
+double categorical_hash(std::string const &value);
 
 #endif
diff --git a/tile.cpp b/tile.cpp
index 9a487f12..528741b8 100644
--- a/tile.cpp
+++ b/tile.cpp
@@ -241,7 +241,6 @@ static int metacmp(const std::vector<long long> &keys1, const std::vector<long l
 }
 
 double get_interestingness(const serial_feature *sf, const std::map<std::string, layermap_entry> *merged_layermaps, std::string const &layername) {
-	double interestingness = 0;
 	auto layer = merged_layermaps->find(layername);
 	if (layer == merged_layermaps->end()) {
 		fprintf(stderr, "Can't find layer %s\n", layername.c_str());
@@ -250,24 +249,35 @@ double get_interestingness(const serial_feature *sf, const std::map<std::string,
 
 	for (size_t i = 0; i < sf->keys.size(); i++) {
 		mvt_value key = retrieve_string(sf->keys[i], sf->stringpool, NULL);
-		mvt_value value = retrieve_string(sf->values[i], sf->stringpool, NULL);
 
-#if 0
-		// this needs to find the layer before it can find the attribute
-		auto lm = merged_layermaps->find(key.string_value);
-		if (lm != merged_layermaps->end()) {
-			// how categorical is it?
+		auto const tass = layer->second.file_keys.find(key.string_value);
+		if (tass != layer->second.file_keys.end()) {
+			// mvt_value value = retrieve_string(sf->values[i], sf->stringpool, NULL);
 
-                        double xd = lm->second.xsum / lm->second.count;
-                        double yd = lm->second.ysum / lm->second.count;
-                        double d = sqrt(xd * xd + yd * yd);
+                        double xd = tass->second.xsum / tass->second.count;
+                        double yd = tass->second.ysum / tass->second.count;
 
-			
+			// This will be 0 for something that is not categorical at all,
+			// 1 for something that is so categorical that there is only one value,
+			// and above 0.3 or so for something that is more reasonably categorical.
+                        double categoricality = sqrt(xd * xd + yd * yd);
+
+			if (categoricality != 1) {
+				double angle = categorical_hash(sf->stringpool + sf->values[i] + 1);
+				xd -= cos(angle);
+				yd -= sin(angle);
+
+				// This will be 0 for something that matches the single categorical value,
+				// almost 2 for something that is an extreme categorical outlier,
+				// or somewhere in between for more reasonably categorical values.
+				double interestingness = sqrt(xd * xd + yd * yd);
+
+				printf("%f: %s %s\n", categoricality * interestingness, key.string_value.c_str(), sf->stringpool + sf->values[i] + 1);
+			}
 		}
-#endif
 	}
 
-	return interestingness;
+	return 0;
 }
 
 static mvt_value find_attribute_value(const serial_feature *sf, std::string key, const std::map<std::string, layermap_entry> *merged_layermaps, std::string const &layername) {