Add --generate-variable-depth-tile-pyramid option (#251)

* Track output position at the file level instead of within each tile

* Track file position where the child tile data begins

* Add option and document its intended behavior

* Changing the detail loop to account for stopping early

* I forgot I already added an option for this

* Stop early if we can make a complete tile

* Add a test of zoom truncation with limited feature count

* Forgot to commit the actual code change

* Make room for a vertex count in the header of each serialized tile

* Estimate tile complexity; don't try truncating when unlikely to work

* Be more conservative, because ever retrying a tile is a big speed hit

* If stopping early, don't simplify or clean; leave that to overzoom

* Add tiny polygon reduction / dust to overzoom

* Don't try to stop early in the children if we dropped anything by rate

* Fflush here too before pwriting

* Don't stop early if we ended up dropping any features.

Rework the can-the-next-zoom-stop-early logic to avoid going
one zoom further than needed.

* Fix warning

* Fix warnings

* Oops, checking for the wrong expected return value

* Cleanup from adding line simplification in overzoom

* Current (wrong) behavior when combining coalescing and truncating

* Keep a list of parent tiles to skip rather than truncating

* Now the coalesced tiles in z12 get children in z13

* Don't double-count feature dropping when the zoom level is retried

* Correct README description

* Remove todo about special case below basezoom, which is accounted for

* Be a little more aggressive in drop-densest determination

* Scale tile feature limit for megatiles in the same way as byte limit

* Fully deprecate -detect-shared-borders into an alias

* Track the distances found in the douglas-peucker recursion

* Serialize and deserialize the distance with the vertices

* Revert "Serialize and deserialize the distance with the vertices"

This reverts commit 753f1b7909.

* Revert "Track the distances found in the douglas-peucker recursion"

This reverts commit e5361f8c22.

* Revert "Fully deprecate -detect-shared-borders into an alias"

This reverts commit 0698aeb766.

* Better tracking of whether we failed to make a full-detail tile

* Put a bloom filter in front of the binary search for shared nodes

* Forgot to take out this printf

* Improve dispatch of tiling tasks

* Still dispatch the biggest tasks first

* Track zoom truncation in the strategies list in the tileset metadata

* Prescan for small deltas before doing proper simplification

* Revert "Prescan for small deltas before doing proper simplification"

This reverts commit d1d8238b83.

* Update version and changelog

* Rename to --generate-variable-depth-tile-pyramid

tile.hpp

@@ -18,6 +18,7 @@ struct atomic_strategy {
 	std::atomic<size_t> coalesced_as_needed;
 	std::atomic<size_t> detail_reduced;
 	std::atomic<size_t> tiny_polygons;
+	std::atomic<size_t> truncated_zooms;
 
 	atomic_strategy()
 	    : dropped_by_rate(0),
@@ -25,8 +26,11 @@ struct atomic_strategy {
 	      dropped_as_needed(0),
 	      coalesced_as_needed(0),
 	      detail_reduced(0),
-	      tiny_polygons(0) {
+	      tiny_polygons(0),
+	      truncated_zooms(0) {
 	}
+
+	void add_from(struct strategy const &src);
 };
 
 struct strategy {
@@ -35,9 +39,11 @@ struct strategy {
 	size_t dropped_as_needed = 0;
 	size_t coalesced_as_needed = 0;
 	size_t detail_reduced = 0;
+	size_t tiny_polygons = 0;
+	size_t truncated_zooms = 0;
+
 	size_t tile_size = 0;
 	size_t feature_count = 0;
-	size_t tiny_polygons = 0;
 
 	strategy(const atomic_strategy &s, size_t ts, size_t fc) {
 		dropped_by_rate = s.dropped_by_rate;
@@ -48,6 +54,7 @@ struct strategy {
 		tile_size = ts;
 		feature_count = fc;
 		tiny_polygons = s.tiny_polygons;
+		truncated_zooms = s.truncated_zooms;
 	}
 
 	strategy() = default;
@@ -55,7 +62,7 @@ struct strategy {
 
 // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
 
-int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, int basezoom, double droprate, std::vector<std::string> const &unidecode_data);
+int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector<std::string> const &unidecode_data);
 
 int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);