Add an all-mvt_value attribute accumulation path

attribute.cpp

@@ -207,3 +207,111 @@ void preserve_attribute(attribute_op const &op, std::string const &key, serial_v
 	full_keys.push_back(key);
 	full_values.push_back(sv);
 }
+
+void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector<std::string> &full_keys, std::vector<mvt_value> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state) {
+	for (size_t i = 0; i < full_keys.size(); i++) {
+		if (key == full_keys[i]) {
+			switch (op) {
+			case op_sum:
+				full_values[i] = mvt_value(full_values[i].to_double() + val.to_double());
+				return;
+
+			case op_product:
+				full_values[i] = mvt_value_to_double(full_values[i].to_double() * val.to_double());
+				return;
+
+			case op_max: {
+				double existing = full_values[i].to_double();
+				double maybe = val.to_double();
+				if (maybe > existing) {
+					full_values[i] = val;
+				}
+				return;
+			}
+
+			case op_min: {
+				double existing = full_values[i].to_double();
+				double maybe = val.to_double();
+				if (maybe < existing) {
+					full_values[i] = val;
+				}
+				return;
+			}
+
+			case op_mean: {
+				auto state = attribute_accum_state.find(key);
+				if (state == attribute_accum_state.end()) {
+					accum_state s;
+					s.sum = full_values[i].to_double() + val.to_double();
+					s.count = 2;
+					attribute_accum_state.insert(std::pair<std::string, accum_state>(key, s));
+
+					full_values[i] = mvt_value(s.sum / s.count);
+				} else {
+					state->second.sum += val.to_double();
+					state->second.count += 1;
+
+					full_values[i] = mvt_value(state->second.sum / state->second.count);
+				}
+				return;
+			}
+
+			case op_concat:
+				full_values[i].set_string_value(full_values[i].toString() + val.toString());
+				return;
+
+			case op_comma:
+				full_values[i].set_string_value(full_values[i].toString() + "," + val.toString());
+				return;
+
+			case op_count: {
+				auto state = attribute_accum_state.find(key);
+				if (state == attribute_accum_state.end()) {  // not already present
+					accum_state s;
+					s.count = 2;
+					attribute_accum_state.insert(std::pair<std::string, accum_state>(key, s));
+
+					full_values[i] = mvt_value(s.count);
+				} else {  // already present, incrementing
+					state->second.count += 1;
+					full_values[i] = mvt_value(state->second.count);
+				}
+				return;
+			}
+			}
+		}
+	}
+
+	// not found, so we are making a new value
+
+	mvt_value v;
+	switch (op) {
+	case op_sum:
+	case op_max:
+	case op_min:
+		v = val;
+		break;
+
+	case op_count: {
+		auto state = attribute_accum_state.find(key);
+		if (state == attribute_accum_state.end()) {  // not already present
+			accum_state s;
+			s.count = 1;
+			attribute_accum_state.insert(std::pair<std::string, accum_state>(key, s));
+
+			v = mvt_value(s.count);
+		} else {  // already present, incrementing
+			fprintf(stderr, "preserve_attribute: can't happen (count)\n");
+			exit(EXIT_IMPOSSIBLE);
+		}
+		break;
+	}
+
+	default:
+		fprintf(stderr, "can't happen: operation that isn't used by --accumulate-numeric-attributes\n");
+		exit(EXIT_IMPOSSIBLE);
+	}
+
+	full_keys.push_back(key);
+	full_values.push_back(v);
+}

attribute.hpp

@@ -4,6 +4,7 @@
 #include <vector>
 #include <unordered_map>
 #include <map>
+#include "mvt.hpp"
 
 enum attribute_op {
 	op_sum,
@@ -26,6 +27,7 @@ struct serial_val;
 void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, std::string name, std::string type);
 void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, const char *arg, char **argv);
 void preserve_attribute(attribute_op const &op, const std::string &key, serial_val const &val, std::vector<std::string> &full_keys, std::vector<serial_val> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state);
+void preserve_attribute(attribute_op const &op, std::string const &key, mvt_value const &val, std::vector<std::string> &full_keys, std::vector<mvt_value> &full_values, std::unordered_map<std::string, accum_state> &attribute_accum_state);
 
 extern std::map<std::string, attribute_op> numeric_operations;
 

clip.cpp

@@ -1165,7 +1165,7 @@ static void add_mean(mvt_feature &feature, mvt_layer &layer, std::string const &
 // accumulate :sum:, :min:, :max:, and :count: versions of the specified attribute
 static void preserve_numeric(const std::string &key, const mvt_value &val,			  // numeric attribute being accumulated
 			     std::vector<std::string> &full_keys,				  // keys of feature being accumulated onto
-			     std::vector<serial_val> &full_values,				  // values of features being accumulated onto
+			     std::vector<mvt_value> &full_values,				  // values of features being accumulated onto
 			     const std::string &accumulate_numeric,				  // prefix of accumulations
 			     std::set<std::string> &keys,					  // key presence in the source feature
 			     std::map<std::string, size_t> &numeric_out_field,			  // key index in the output feature
@@ -1218,16 +1218,13 @@ static void preserve_numeric(const std::string &key, const mvt_value &val,			  /
 					if (op.second == op_count) {
 						if (starting_from_accumulation) {
 							// copy our count
-							full_values.push_back(mvt_value_to_serial_val(val));
+							full_values.push_back(val);
 						} else {
 							// new count of 1
-							serial_val sv;
-							sv.type = mvt_double;
-							sv.s = "1";
-							full_values.push_back(sv);
+							full_values.push_back(mvt_value(1));
 						}
 					} else {
-						full_values.push_back(mvt_value_to_serial_val(val));
+						full_values.push_back(val);
 					}
 				} else {
 					// exists unprefixed, so copy it, and then accumulate on our value
@@ -1235,19 +1232,18 @@ static void preserve_numeric(const std::string &key, const mvt_value &val,			  /
 					full_keys.push_back(prefixed);
 
 					if (op.second == op_count) {
-						serial_val sv;
-						sv.type = mvt_double;
+						mvt_value v;
 						if (starting_from_accumulation) {
 							// sum our count onto the existing 1
-							sv.s = std::to_string(1 + mvt_value_to_long_long(val));
+							v = mvt_value(1 + mvt_value_to_long_long(val));
 						} else {
 							// sum our 1 onto the existing 1
-							sv.s = "2";
+							v = mvt_value(2);
 						}
-						full_values.push_back(sv);
+						full_values.push_back(v);
 					} else {
 						full_values.push_back(full_values[out_attr->second]);
-						preserve_attribute(op.second, prefixed, mvt_value_to_serial_val(val), full_keys, full_values, attribute_accum_state);
+						preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state);
 					}
 				}
 			} else {
@@ -1255,12 +1251,12 @@ static void preserve_numeric(const std::string &key, const mvt_value &val,			  /
 				if (op.second == op_count) {
 					if (starting_from_accumulation) {
 						// sum our count onto the existing count
-						full_values[prefixed_attr->second].s = std::to_string(atoll(full_values[prefixed_attr->second].s.c_str()) + mvt_value_to_long_long(val));
+						full_values[prefixed_attr->second] = mvt_value(mvt_value_to_long_long(full_values[prefixed_attr->second]) + mvt_value_to_long_long(val));
 					} else {
-						full_values[prefixed_attr->second].s = std::to_string(atoll(full_values[prefixed_attr->second].s.c_str()) + 1);
+						full_values[prefixed_attr->second] = mvt_value(mvt_value_to_long_long(full_values[prefixed_attr->second]) + 1);
 					}
 				} else {
-					preserve_attribute(op.second, prefixed, mvt_value_to_serial_val(val), full_keys, full_values, attribute_accum_state);
+					preserve_attribute(op.second, prefixed, val, full_keys, full_values, attribute_accum_state);
 				}
 			}
 		}
@@ -1293,7 +1289,6 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
 			std::set<std::string> const &exclude,
 			std::vector<std::string> const &exclude_prefix,
 			std::unordered_map<std::string, attribute_op> const &attribute_accum,
-			std::shared_ptr<std::string> const &tile_stringpool,
 			std::string const &accumulate_numeric) {
 	// Add geometry to output feature
 
@@ -1315,13 +1310,13 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
 
 		if (attribute_accum.size() > 0 || accumulate_numeric.size() > 0) {
 			// convert the attributes of the output feature
-			// from mvt_value to serial_val so they can have
+			// from layer references to a vector so they can have
 			// attributes from the other features of the
 			// multiplier cluster accumulated onto them
 
 			std::unordered_map<std::string, accum_state> attribute_accum_state;
 			std::vector<std::string> full_keys;
-			std::vector<serial_val> full_values;
+			std::vector<mvt_value> full_values;
 			std::map<std::string, size_t> numeric_out_field;
 
 			for (size_t i = 0; i + 1 < features[0].tags.size(); i += 2) {
@@ -1330,12 +1325,12 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
 				if (f != attribute_accum.end()) {
 					// this attribute has an accumulator, so convert it
 					full_keys.push_back(features[0].layer->keys[features[0].tags[i]]);
-					full_values.push_back(mvt_value_to_serial_val(features[0].layer->values[features[0].tags[i + 1]]));
+					full_values.push_back(features[0].layer->values[features[0].tags[i + 1]]);
 				} else if (accumulate_numeric.size() > 0 && features[0].layer->values[features[0].tags[i + 1]].is_numeric()) {
 					// convert numeric for accumulation
 					numeric_out_field.emplace(key, full_keys.size());
 					full_keys.push_back(key);
-					full_values.push_back(mvt_value_to_serial_val(features[0].layer->values[features[0].tags[i + 1]]));
+					full_values.push_back(features[0].layer->values[features[0].tags[i + 1]]);
 				} else {
 					// otherwise just tag it directly onto the output feature
 					if (should_keep(features[0].layer->keys[features[0].tags[i]], keep, exclude, exclude_prefix)) {
@@ -1361,7 +1356,7 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
 
 					auto f = attribute_accum.find(key);
 					if (f != attribute_accum.end()) {
-						serial_val val = mvt_value_to_serial_val(features[i].layer->values[features[i].tags[j + 1]]);
+						mvt_value val = features[i].layer->values[features[i].tags[j + 1]];
 						preserve_attribute(f->second, key, val, full_keys, full_values, attribute_accum_state);
 					} else if (accumulate_numeric.size() > 0) {
 						const mvt_value &val = features[i].layer->values[features[i].tags[j + 1]];
@@ -1379,7 +1374,7 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
 
 			for (size_t i = 0; i < full_keys.size(); i++) {
 				if (should_keep(full_keys[i], keep, exclude, exclude_prefix)) {
-					outlayer.tag(outfeature, full_keys[i], stringified_to_mvt_value(full_values[i].type, full_values[i].s.c_str(), tile_stringpool));
+					outlayer.tag(outfeature, full_keys[i], full_values[i]);
 				}
 			}
 
@@ -1570,7 +1565,6 @@ mvt_tile assign_to_bins(mvt_tile &features,
 	outlayer.name = features.layers[0].name;
 
 	std::vector<std::vector<tile_feature>> outfeatures;
-	std::shared_ptr<std::string> tile_stringpool = std::make_shared<std::string>();
 
 	for (auto &e : events) {
 		if (e.kind == index_event::ENTER) {
@@ -1680,7 +1674,7 @@ mvt_tile assign_to_bins(mvt_tile &features,
 		if (outfeatures[i].size() > 1) {
 			feature_out(outfeatures[i], outlayer,
 				    keep, exclude, exclude_prefix, attribute_accum,
-				    tile_stringpool, accumulate_numeric);
+				    accumulate_numeric);
 			mvt_feature &nfeature = outlayer.features.back();
 			mvt_value val;
 			val.type = mvt_uint;
@@ -1715,7 +1709,6 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
 		     std::string const &accumulate_numeric) {
 	mvt_tile outtile;
-	std::shared_ptr<std::string> tile_stringpool = std::make_shared<std::string>();
 
 	for (auto const &tile : tiles) {
 		for (auto const &layer : tile.tile.layers) {
@@ -1840,7 +1833,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 
 				if (flush_multiplier_cluster) {
 					if (pending_tile_features.size() > 0) {
-						feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, tile_stringpool, accumulate_numeric);
+						feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric);
 						pending_tile_features.clear();
 					}
 				}
@@ -1897,7 +1890,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 			}
 
 			if (pending_tile_features.size() > 0) {
-				feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, tile_stringpool, accumulate_numeric);
+				feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric);
 				pending_tile_features.clear();
 			}
 

mvt.hpp

@@ -76,6 +76,9 @@ enum mvt_value_type {
 	mvt_no_such_key,
 };
 
+struct mvt_value;
+double mvt_value_to_double(mvt_value const &v);
+
 struct mvt_value {
 	mvt_value_type type;
 	std::shared_ptr<std::string> s;
@@ -126,6 +129,10 @@ struct mvt_value {
 		       type == mvt_sint;
 	}
 
+	double to_double() const {
+		return mvt_value_to_double(*this);
+	}
+
 	bool operator<(const mvt_value &o) const;
 	bool operator==(const mvt_value &o) const;
 	std::string toString() const;
@@ -134,6 +141,11 @@ struct mvt_value {
 		this->type = mvt_double;
 		this->numeric_value.double_value = 0;
 	}
+
+	mvt_value(double v) {
+		this->type = mvt_double;
+		this->numeric_value.double_value = v;
+	}
 };
 
 template <>
@@ -201,7 +213,6 @@ int dezig(unsigned n);
 
 mvt_value stringified_to_mvt_value(int type, const char *s, std::shared_ptr<std::string> const &tile_stringpool);
 long long mvt_value_to_long_long(mvt_value const &v);
-double mvt_value_to_double(mvt_value const &v);
 
 bool is_integer(const char *s, long long *v);
 bool is_unsigned_integer(const char *s, unsigned long long *v);