Move attribute type coercion out of parsing and into serialization

geojson.cpp

@@ -99,7 +99,7 @@ static long long scale_geometry(struct serialization_state *sst, long long *bbox
 	return geom.size();
 }
 
-int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string layername, std::map<std::string, int> const *attribute_types) {
+int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string layername) {
 	json_object *geometry_type = json_hash_get(geometry, "type");
 	if (geometry_type == NULL) {
 		static int warned = 0;
@@ -219,7 +219,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
 
 			int type = -1;
 			std::string val;
-			stringify_value(properties->values[i], type, val, sst->fname, sst->line, feature, properties->keys[i]->string, attribute_types);
+			stringify_value(properties->values[i], type, val, sst->fname, sst->line, feature, properties->keys[i]->string);
 
 			if (type >= 0) {
 				metakey[m] = properties->keys[i]->string;
@@ -406,10 +406,18 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
 			if (sst->include->count(sf.full_keys[i]) == 0) {
 				sf.full_keys[i] = "";
 				sf.m--;
+				continue;
 			}
 		} else if (sst->exclude->count(sf.full_keys[i]) != 0) {
 			sf.full_keys[i] = "";
 			sf.m--;
+			continue;
+		}
+
+		coerce_value(sf.full_keys[i], sf.full_values[i].type, sf.full_values[i].s, sst->attribute_types);
+		if (sf.full_values[i].type == mvt_null) {
+			sf.full_keys[i] = "";
+			sf.m--;
 		}
 	}
 
@@ -501,7 +509,7 @@ void check_crs(json_object *j, const char *reading) {
 	}
 }
 
-void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername, std::map<std::string, int> const *attribute_types) {
+void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername) {
 	long long found_hashes = 0;
 	long long found_features = 0;
 	long long found_geometries = 0;
@@ -569,7 +577,7 @@ void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::
 				}
 				found_geometries++;
 
-				serialize_geojson_feature(sst, j, NULL, NULL, layer, NULL, j, layername, attribute_types);
+				serialize_geojson_feature(sst, j, NULL, NULL, layer, NULL, j, layername);
 				json_free(j);
 				continue;
 			}
@@ -612,10 +620,10 @@ void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::
 		if (geometries != NULL) {
 			size_t g;
 			for (g = 0; g < geometries->length; g++) {
-				serialize_geojson_feature(sst, geometries->array[g], properties, id, layer, tippecanoe, j, layername, attribute_types);
+				serialize_geojson_feature(sst, geometries->array[g], properties, id, layer, tippecanoe, j, layername);
 			}
 		} else {
-			serialize_geojson_feature(sst, geometry, properties, id, layer, tippecanoe, j, layername, attribute_types);
+			serialize_geojson_feature(sst, geometry, properties, id, layer, tippecanoe, j, layername);
 		}
 
 		json_free(j);
@@ -627,7 +635,7 @@ void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::
 void *run_parse_json(void *v) {
 	struct parse_json_args *pja = (struct parse_json_args *) v;
 
-	parse_json(pja->sst, pja->jp, pja->layer, *pja->layername, pja->attribute_types);
+	parse_json(pja->sst, pja->jp, pja->layer, *pja->layername);
 
 	return NULL;
 }

geojson.hpp

@@ -22,7 +22,7 @@ struct parse_json_args {
 struct json_pull *json_begin_map(char *map, long long len);
 void json_end_map(struct json_pull *jp);
 
-void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername, std::map<std::string, int> const *attribute_types);
+void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername);
 void *run_parse_json(void *v);
 
 #endif

main.cpp

@@ -403,11 +403,11 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
 		sst[i].include = include;
 		sst[i].exclude_all = exclude_all;
 		sst[i].basezoom = basezoom;
+		sst[i].attribute_types = attribute_types;
 
 		pja[i].jp = json_begin_map(map + segs[i], segs[i + 1] - segs[i]);
 		pja[i].layer = source;
 		pja[i].layername = &layername;
-		pja[i].attribute_types = attribute_types;
 
 		pja[i].sst = &sst[i];
 
@@ -1382,8 +1382,9 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 				sst.include = include;
 				sst.exclude_all = exclude_all;
 				sst.basezoom = basezoom;
+				sst.attribute_types = attribute_types;
 
-				parse_json(&sst, jp, layer, sources[layer].layer, attribute_types);
+				parse_json(&sst, jp, layer, sources[layer].layer);
 				json_end(jp);
 				overall_offset = layer_seq;
 				checkdisk(reader, CPUS);

mvt.hpp

@@ -62,7 +62,8 @@ enum mvt_value_type {
 	mvt_int,
 	mvt_uint,
 	mvt_sint,
-	mvt_bool
+	mvt_bool,
+	mvt_null,
 };
 
 struct mvt_value {

plugin.cpp

@@ -257,8 +257,7 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
 				int tp = -1;
 				std::string s;
 
-				std::map<std::string, int> nullmap;
+				stringify_value(properties->values[i], tp, s, "Filter output", jp->line, j, "");
-				stringify_value(properties->values[i], tp, s, "Filter output", jp->line, j, "", &nullmap);
 				if (tp >= 0) {
 					mvt_value v = stringified_to_mvt_value(tp, s.c_str());
 					l->second.tag(feature, std::string(properties->keys[i]->string), v);
@@ -487,8 +486,7 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
 				serial_val v;
 				v.type = -1;
 
-				std::map<std::string, int> nullmap;
+				stringify_value(properties->values[i], v.type, v.s, "Filter output", jp->line, j, "");
-				stringify_value(properties->values[i], v.type, v.s, "Filter output", jp->line, j, "", &nullmap);
 
 				if (v.type >= 0) {
 					sf.full_keys.push_back(std::string(properties->keys[i]->string));

read_json.cpp

@@ -106,7 +106,7 @@ void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fna
 	}
 }
 
-void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key, std::map<std::string, int> const *attribute_types) {
+void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key) {
 	if (value != NULL) {
 		int vt = value->type;
 		std::string val;
@@ -125,40 +125,6 @@ void stringify_value(json_object *value, int &type, std::string &stringified, co
 			free((void *) v);  // stringify
 		}
 
-		auto a = (*attribute_types).find(key);
-		if (a != attribute_types->end()) {
-			if (a->second == mvt_string) {
-				vt = JSON_STRING;
-			} else if (a->second == mvt_float) {
-				vt = JSON_NUMBER;
-				val = std::to_string(atof(val.c_str()));
-			} else if (a->second == mvt_int) {
-				vt = JSON_NUMBER;
-				if (val.size() == 0) {
-					val = "0";
-				}
-
-				for (size_t ii = 0; ii < val.size(); ii++) {
-					char c = val[ii];
-					if (c < '0' || c > '9') {
-						val = std::to_string(round(atof(val.c_str())));
-						break;
-					}
-				}
-			} else if (a->second == mvt_bool) {
-				if (val == "false" || val == "0" || val == "null" || val.size() == 0) {
-					vt = JSON_FALSE;
-					val = "false";
-				} else {
-					vt = JSON_TRUE;
-					val = "true";
-				}
-			} else {
-				fprintf(stderr, "Can't happen: attribute type %d\n", a->second);
-				exit(EXIT_FAILURE);
-			}
-		}
-
 		if (vt == JSON_STRING) {
 			type = mvt_string;
 			stringified = val;
@@ -175,10 +141,47 @@ void stringify_value(json_object *value, int &type, std::string &stringified, co
 			type = mvt_bool;
 			stringified = val;
 		} else if (vt == JSON_NULL) {
-			;
+			type = mvt_null;
+			stringified = "null";
 		} else {
 			type = mvt_string;
 			stringified = val;
 		}
 	}
 }
+
+void coerce_value(std::string const &key, int &vt, std::string &val, std::map<std::string, int> const *attribute_types) {
+	auto a = (*attribute_types).find(key);
+	if (a != attribute_types->end()) {
+		if (a->second == mvt_string) {
+			vt = mvt_string;
+		} else if (a->second == mvt_float) {
+			vt = mvt_double;
+			val = std::to_string(atof(val.c_str()));
+		} else if (a->second == mvt_int) {
+			vt = mvt_double;
+			if (val.size() == 0) {
+				val = "0";
+			}
+
+			for (size_t ii = 0; ii < val.size(); ii++) {
+				char c = val[ii];
+				if (c < '0' || c > '9') {
+					val = std::to_string(round(atof(val.c_str())));
+					break;
+				}
+			}
+		} else if (a->second == mvt_bool) {
+			if (val == "false" || val == "0" || val == "null" || val.size() == 0) {
+				vt = mvt_bool;
+				val = "false";
+			} else {
+				vt = mvt_bool;
+				val = "true";
+			}
+		} else {
+			fprintf(stderr, "Can't happen: attribute type %d\n", a->second);
+			exit(EXIT_FAILURE);
+		}
+	}
+}

read_json.hpp

@@ -13,4 +13,5 @@ extern int mb_geometry[GEOM_TYPES];
 void json_context(json_object *j);
 void parse_geometry(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature);
 
-void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key, std::map<std::string, int> const *attribute_types);
+void stringify_value(json_object *value, int &type, std::string &stringified, const char *reading, int line, json_object *feature, std::string const &key);
+void coerce_value(std::string const &key, int &vt, std::string &val, std::map<std::string, int> const *attribute_types);

serial.hpp

@@ -121,6 +121,7 @@ struct serialization_state {
 
 	std::map<std::string, layermap_entry> *layermap;
 
+	std::map<std::string, int> const *attribute_types;
 	std::set<std::string> *exclude;
 	std::set<std::string> *include;
 	int exclude_all;