Improve tiling speed (#195)

* Add a way to run tippecanoe single-threaded for profiling

* Do less work when the tilestats sample values list is already full

* Save a copy when retrieving the attribute key

* Fewer atomic operations

* Move string hashing from mbtiles to text

* Only do approximate attribute deduplication when writing tiles

* Feature dropping tests are sensitive to exact tile size

* All tile creators now create a string pool for the tile

* Features clipped away to nothing should not participate in that tile

* Revert "Only do approximate attribute deduplication when writing tiles"

This reverts commit c42b34b498.

* Also revert the related test changes

* Revert "Revert "Only do approximate attribute deduplication when writing tiles""

This reverts commit 18509876c3.

* Be more specific about the string hash function

* Use fnv1a instead of std::hash for everything

* Reduce the chance of hash collisions

* Stick a hash search on the front of the tree search in addpool

* Eliminate repeated hashing of the same string

* Switch instead of ifs in json parsing

* A few more cases to populate the hash in addpool

* Store the hash in the tree instead of recalculating

* Add explanatory comment for mysterious argument

* Fewer copies in attribute stringification

* Clean up ancient weirdness in JSON attribute stringification

* More serial_val cleanup

* Pass a serial_feature to rewrite instead of many broken-down arguments

* Get rid of the multiple geometries within `partial`

* Revert "Pass a serial_feature to rewrite instead of many broken-down arguments"

This reverts commit 6f4ab9b725.

* Goodbye, struct coalesce

* Revert "Features clipped away to nothing should not participate in that tile"

This reverts commit 124462fbdc.

* Migrating fields from partial to serial_feature

* Name reconciliation between serial_feature and partial

* Replace struct partial with an augmented serial_feature

* Fix some overzealous search-and-replace renaming

* Don't say struct so often

* Remove more of the former partial construction

* Commenting and cleaning up

* Trying again to avoid all these arguments to rewrite

* I swear I did this same thing before and it didn't work.

* More rewrite cleanup

* Exile --detect-shared-borders to its own file

* Add missing headers

* More commenting and cleanup

* More comments

* Sprinkle consts around

* Emplacing and std::moving

* More cleanup

* That shouldn't have worked after a std::move

* Don't need to allocate memory to compare keys

* Reduce use of the global string pool in tiling

* Another avoidable mvt_value construction

* Further reduction to explicit string pool passing

* These reverses are no longer optimizations

* These layernames can all be references

* Don't drag an unused layername string around with every feature

* Heed a compiler warning about potential buffer overflow

* Fix my confusion about which feature's string pool is relevant

* Avoid some unnecessary allocations in attribute accumulation

* Maybe faster serialization?

* Eliminate a comparison

* Do the same here

* Save a couple of allocations when parsing numbers in JSON

* Immediately assign features to layers instead of subdividing later

* Maintain tilestats for tippecanoe:retain_points_multiplier_sequence

* Crunch out more duplicate attribute values when writing out the tile

* Do tilestats for tippecanoe:retain_points_multiplier_first too

* Shell filters need to be real threads, even if nothing else does

* Simplify tippecanoe_minzoom/maxzoom representation

* Update version and changelog

serial.hpp

@@ -25,22 +25,30 @@ void serialize_ulong_long(std::string &out, unsigned long long n);
 void serialize_byte(std::string &out, signed char n);
 void serialize_uint(std::string &out, unsigned n);
 
-void deserialize_int(char **f, int *n);
-void deserialize_long_long(char **f, long long *n);
-void deserialize_ulong_long(char **f, unsigned long long *n);
-void deserialize_uint(char **f, unsigned *n);
-void deserialize_byte(char **f, signed char *n);
+void deserialize_int(const char **f, int *n);
+void deserialize_long_long(const char **f, long long *n);
+void deserialize_ulong_long(const char **f, unsigned long long *n);
+void deserialize_uint(const char **f, unsigned *n);
+void deserialize_byte(const char **f, signed char *n);
 
 // This is the main representation of attribute values in memory and
 // in the string pool. The type is one of the mvt_value type (mvt_string,
 // mvt_double, mvt_bool, or mvt_null). Note that all numeric values,
 // whether integer or floating point, use mvt_double here.
 struct serial_val {
-	int type = 0;
-	std::string s = "";
+	int type;
+	std::string s;
 
 	bool operator<(const serial_val &o) const;
 	bool operator!=(const serial_val &o) const;
+
+	serial_val() {
+		type = 0;
+	}
+
+	serial_val(int t, const std::string &val)
+	    : type(t), s(val) {
+	}
 };
 
 struct serial_feature {
@@ -54,31 +62,58 @@ struct serial_feature {
 	bool has_id = false;
 	unsigned long long id = 0;
 
-	bool has_tippecanoe_minzoom = false;
-	int tippecanoe_minzoom = 0;
-
-	bool has_tippecanoe_maxzoom = false;
-	int tippecanoe_maxzoom = 0;
+	int tippecanoe_minzoom = -1;
+	int tippecanoe_maxzoom = -1;
 
 	drawvec geometry = drawvec();
 	unsigned long long index = 0;
 	unsigned long long label_point = 0;
 	long long extent = 0;
 
+	// These fields are not directly serialized, but are used
+	// to create the keys and values references into the string pool
+	// during initial serialization
+
+	std::vector<std::string> full_keys{};
+	std::vector<serial_val> full_values{};
+
+	// These fields are generated from full_keys and full_values
+	// during initial serialization and then replace the string
+	// representations:
+
 	std::vector<long long> keys{};
 	std::vector<long long> values{};
 
-	// XXX This isn't serialized. Should it be here?
+	// These fields are used during tiling,
+	// but are not serialized and are not expected
+	// to be provided by frontends:
+
 	long long bbox[4] = {0, 0, 0, 0};
-	std::vector<std::string> full_keys{};
-	std::vector<serial_val> full_values{};
-	std::string layername = "";
-	bool dropped = false;
-	drawvec edge_nodes;
+	bool dropped = false;  // was this feature dropped by rate?
+	drawvec edge_nodes;    // what nodes at the tile edge were added during clipping?
+
+	bool reduced;	   // is polygon dust
+	bool coalesced;	   // was coalesced from multiple features
+	int line_detail;   // current tile resolution being used for simplification
+	int extra_detail;  // extra tile resolution to retain in output
+	int maxzoom;
+	double spacing;				       // feature spacing for --calculate-feature-density
+	double simplification;			       // simplification level at this zoom level
+	std::vector<ssize_t> arc_polygon;	       // used in --detect-shared-borders
+	ssize_t renamed;			       // used in --detect-shared-borders logic
+	long long clustered;			       // does this feature need the clustered/point_count attributes?
+	const char *stringpool;			       // string pool for keys/values lookup
+	std::shared_ptr<std::string> tile_stringpool;  // string pool for mvt_value construction
+	std::set<std::string> need_tilestats;
+	std::unordered_map<std::string, accum_state> attribute_accum_state;
+
+	int z;	// tile being produced
+	int tx;
+	int ty;
 };
 
 std::string serialize_feature(serial_feature *sf, long long wx, long long wy);
-serial_feature deserialize_feature(std::string &geoms, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
+serial_feature deserialize_feature(std::string const &geoms, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
 
 struct reader {
 	int poolfd = -1;
@@ -101,14 +136,15 @@ struct reader {
 	std::atomic<long long> nodepos;
 
 	long long file_bbox[4] = {0, 0, 0, 0};
-
 	long long file_bbox1[4] = {0xFFFFFFFF, 0xFFFFFFFF, 0, 0};	      // standard -180 to 180 world plane
 	long long file_bbox2[4] = {0x1FFFFFFFF, 0xFFFFFFFF, 0x100000000, 0};  // 0 to 360 world plane
 
 	struct stat geomst {};
-
 	char *geom_map = NULL;
 
+	std::vector<ssize_t> key_dedup = std::vector<ssize_t>(655536, -1);
+	std::vector<ssize_t> value_dedup = std::vector<ssize_t>(655536, -1);
+
 	reader()
 	    : geompos(0), indexpos(0), vertexpos(0), nodepos(0) {
 	}
@@ -227,7 +263,7 @@ struct node {
 
 int nodecmp(const void *void1, const void *void2);
 
-int serialize_feature(struct serialization_state *sst, serial_feature &sf);
+int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername);
 void coerce_value(std::string const &key, int &vt, std::string &val, std::unordered_map<std::string, int> const *attribute_types);
 
 #endif