Merge remote-tracking branch 'origin/main' into jsonpull-rename

.github/workflows/test.yml

@@ -19,3 +19,20 @@ jobs:
         run: brew install sqlite3
       - run: uname -a; BUILDTYPE=${{ matrix.version }} make
       - run: make test
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      # Pinned, because different go-md2man versions produce different roff
+      # for the same input, which would make this check fail spuriously.
+      - name: Install go-md2man
+        run: go install github.com/cpuguy83/go-md2man/v2@v2.0.7
+      - name: Regenerate the man page
+        run: PATH="$PATH:$(go env GOPATH)/bin" make -B docs
+      - name: Check that the man page is up to date with README.md
+        run: |
+          git diff --exit-code man/tippecanoe.1 || {
+            echo "::error::man/tippecanoe.1 is out of date. Run 'make docs' and commit the result."
+            exit 1
+          }

.gitignore

@@ -38,6 +38,9 @@ tippecanoe-json-tool
 tippecanoe-overzoom
 unit
 
+# Left behind if man page generation fails
+man/tippecanoe.1.tmp
+
 # Tests
 tests/**/*.mbtiles
 tests/**/*.check

CHANGELOG.md

@@ -1,3 +1,71 @@
+# 2.81.0
+
+* Add `--drop-by-attribute-as-needed=`*attribute* to drop the features with
+  the lowest values of a numeric attribute from oversized tiles, and
+  `--drop-by-attribute-order=desc` to drop the highest values instead.
+  Features exactly at the threshold are kept rather than dropped. (#384, #385)
+* Add `--exclude-all-tile-geometries` to tile-join, to produce tiles that
+  carry only attributes. (#382)
+* Generate each tool's usage message from the same option table that
+  `getopt_long()` reads, so the hand-written lists in tile-join,
+  tippecanoe-overzoom, tippecanoe-json-tool, tippecanoe-decode, and
+  tippecanoe-enumerate can no longer fall behind the options actually
+  accepted. Options previously reachable only by their short names are now
+  listed. tippecanoe-overzoom reports a missing `-o` instead of passing a
+  null pointer to `fopen()`, and tippecanoe prints its usage when run with
+  no arguments. (#409)
+* Fix the radix sort used by `--prefer-radix-sort`. A bucket written out
+  directly rather than through the merge was written one byte longer than
+  its length prefix claimed, desynchronizing everything read from the
+  geometry after it. Subdividing could also recurse forever once it ran out
+  of files to split with, shifting by the full width of the index and
+  writing past the end of the arrays of buckets. Radix-sorted output is now
+  checked against the in-memory sort rather than against a stored copy. (#404)
+* Read FlatGeobuf integer and float properties as numbers. They were tagged
+  with types that the rest of tippecanoe does not treat as numeric, so they
+  were reported in tilestats as "mixed", with quoted values and no min or
+  max, and warned when used as a feature ID. ULong properties are now also
+  read as unsigned rather than signed. (#395)
+* Respect the `-t` temporary directory option in sorting operations, which
+  previously always used the system temporary directory. (#368)
+* Keep `--generate-variable-depth-tile-pyramid` from silently dropping
+  features whose explicit per-feature `minzoom` is deeper than the zoom at
+  which their region becomes a leaf. Such a feature was excluded from the
+  leaf tile while its children were never generated, so it appeared at no
+  zoom at all. (#397, #399)
+* Drop a polygon hole that no remaining ring can parent, instead of failing
+  the whole run. Degenerate input could abort tiling over a single
+  unrepresentable sliver. (#401)
+* Clamp the feature extent to the `long long` range before converting it,
+  at both ends. The previous `extent <= LLONG_MAX` guard was doubly wrong:
+  `LLONG_MAX` is not representable as a double and rounds up, so an extent
+  at the very top of the range overflowed the conversion and came out as the
+  most negative value rather than the largest, and the guard admitted
+  everything below `LLONG_MIN` as well, which overflowed the other way. Areas
+  are signed, so holes that outweigh their rings can reach the low end. (#406)
+* Initialize the full width of the `mvt_value` numeric union, which left the
+  bytes of the wider unused member indeterminate even though the implicit
+  copy constructor copies the union as a whole. (#406)
+* Replace all variable-length arrays with `std::vector` and `std::string`,
+  and build with `-Wvla`. VLAs are a compiler extension rather than standard
+  C++, and clang warns about every one of them by default. (#406)
+* Remove the unused Dockerfiles, Travis configuration, and lambda
+  directory. (#365)
+* Correct README statements that disagreed with the code. Among them, `-aD`
+  and `-aS` were documented the wrong way round,
+  `--limit-base-zoom-to-maximum-zoom` was given as `-Pb` rather than `-pb`,
+  and the dot-dropping description had both the fraction and the zoom
+  direction backwards: tippecanoe keeps 1/2.5 of the dots at zooms below the
+  base zoom, rather than dropping that share above it. (#410)
+* Generate `man/tippecanoe.1` with go-md2man rather than md2man-roff, which
+  is packaged only as a Ruby gem and so had let the page drift out of date.
+  The page now has a proper header and a NAME section, so `man -k` and
+  `whatis` can find it, and no longer silently drops or mangles text the old
+  converter mishandled. CI checks it against README.md. (#408)
+* Documentation fixes: correct three misspellings in the README and man
+  page, repair the dead All Streets link, and tag more README code blocks
+  with their language. (#375, #391, #400)
+
 # 2.80.0
 
 * Remove undocumented command-line options

Makefile

@@ -11,7 +11,7 @@ CXX := $(CXX)
 CFLAGS := $(CFLAGS) -fPIE -DBUILD_INFO=$(BUILD_INFO)
 CXXFLAGS := $(CXXFLAGS) -std=c++17 -fPIE -DBUILD_INFO=$(BUILD_INFO)
 LDFLAGS := $(LDFLAGS)
-WARNING_FLAGS := -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow
+WARNING_FLAGS := -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow -Wvla
 RELEASE_FLAGS := -O3 -DNDEBUG
 DEBUG_FLAGS := -O0 -DDEBUG -fno-inline-functions -fno-omit-frame-pointer
 
@@ -48,8 +48,44 @@ install: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join tippecanoe-
 uninstall:
 	rm $(PREFIX)/bin/tippecanoe $(PREFIX)/bin/tippecanoe-enumerate $(PREFIX)/bin/tippecanoe-decode $(PREFIX)/bin/tile-join $(MANDIR)/tippecanoe.1 $(PREFIX)/bin/tippecanoe-json-tool
 
+# The man page is generated from README.md by go-md2man, which is packaged for
+# most systems (`brew install go-md2man`, `apt-get install go-md2man`) or can be
+# built with `go install github.com/cpuguy83/go-md2man/v2@v2.0.7`. CI checks that
+# the committed man page matches the README, so you don't have to regenerate it
+# yourself if you don't have go-md2man installed.
+#
+# README.md has no .TH or NAME section of its own, since neither would make sense
+# on GitHub, so prepend them here. go-md2man reads the leading "%" line as the
+# man page's title, section, date, and source. The version deliberately doesn't
+# appear there: it would make this page a build product of version.hpp, so every
+# release would have to regenerate it just to rewrite that one line, and the docs
+# CI job would fail on any version bump that forgot to.
+#
+# Two fixups on the way out:
+#
+#   - README.md's own title heading becomes the second .SH, right below the NAME
+#     section added above, which reads as a stray "tippecanoe" section. Rename it
+#     to DESCRIPTION, where the text under it belongs anyway.
+#   - go-md2man separates paragraphs with a blank line in addition to the .PP
+#     macro, and a blank line is itself a break in roff, so the two together
+#     double-space the whole page. Drop them, except within .EX and .TS blocks,
+#     where a blank line is part of the example or table rather than spacing.
 man/tippecanoe.1: README.md
-	md2man-roff README.md > man/tippecanoe.1
+	{ \
+		echo '% TIPPECANOE 1 "" "tippecanoe"'; \
+		echo; \
+		echo '# NAME'; \
+		echo; \
+		echo 'tippecanoe - build vector tilesets from GeoJSON, FlatGeobuf, or CSV features'; \
+		echo; \
+		cat README.md; \
+	} | go-md2man \
+	  | awk ' \
+		/^\.SH / && ++sh == 2 { print ".SH DESCRIPTION"; next } \
+		/^\.(EX|TS)$$/ { lit = 1 } \
+		/^\.(EE|TE)$$/ { lit = 0 } \
+		lit || !/^$$/ \
+	    ' > $@.tmp && mv $@.tmp $@
 
 PG=
 
@@ -59,25 +95,25 @@ C = $(wildcard *.c) $(wildcard *.cpp)
 INCLUDES = -I/usr/local/include -I. -Iclipper2/include
 LIBS = -L/usr/local/lib
 
-tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o platform.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o
+tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o platform.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o usage.o clipper2/src/clipper.engine.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
-tippecanoe-enumerate: enumerate.o
+tippecanoe-enumerate: enumerate.o usage.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3
 
-tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o
+tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o usage.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
-tile-join: tile-join.o platform.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o
+tile-join: tile-join.o platform.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o usage.o clipper2/src/clipper.engine.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
-tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
+tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o usage.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 unit: unit.o text.o sort.o mvt.o projection.o clip.o attribute.o jsonpull/jsonpull.o evaluator.o read_json.o clipper2/src/clipper.engine.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
-tippecanoe-overzoom: overzoom.o mvt.o clip.o evaluator.o jsonpull/jsonpull.o text.o attribute.o read_json.o projection.o read_json.o clipper2/src/clipper.engine.o
+tippecanoe-overzoom: overzoom.o mvt.o clip.o evaluator.o jsonpull/jsonpull.o text.o attribute.o read_json.o projection.o read_json.o usage.o clipper2/src/clipper.engine.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 -include $(wildcard *.d)
@@ -97,7 +133,7 @@ indent:
 TESTS = $(wildcard tests/*/out/*.json)
 SPACE = $(NULL) $(NULL)
 
-test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test allow-existing-test csv-test layer-json-test pmtiles-test decode-pmtiles-test overzoom-test
+test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test radix-sort-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test allow-existing-test csv-test layer-json-test pmtiles-test decode-pmtiles-test overzoom-test flatgeobuf-test
 	./unit
 
 suffixes = json json.gz
@@ -108,6 +144,7 @@ suffixes = json json.gz
 # %22 for quoted quote
 # %2f for /
 # %3a for :
+# %3d for =
 # %5f for _
 # %7b for {
 
@@ -115,10 +152,11 @@ testargs = \
     $(subst %20,' ',\
         $(subst %22,'"',\
             $(subst %3a,:,\
-                $(subst %2f,/,\
+                $(subst %3d,=,\
-                    $(subst %7b,'{',\
+                    $(subst %2f,/,\
-                        $(subst %5f,'_',\
+                        $(subst %7b,'{',\
-                            $(subst _, ,$(1))))))))
+                            $(subst %5f,'_',\
+                                $(subst _, ,$(1)))))))))
 
 %.json.check:
 	./tippecanoe -q -a@ -f -o $@.mbtiles $(call testargs,$(patsubst %.json.check,%,$(word 4,$(subst /, ,$@)))) $(foreach suffix,$(suffixes),$(sort $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.$(suffix)))) < /dev/null
@@ -132,7 +170,7 @@ nogeobuf = tests/overflow/out/-z0.json $(wildcard tests/stringid/out/*.json)
 geobuf-test: tippecanoe-json-tool $(addsuffix .checkbuf,$(filter-out $(nogeobuf),$(TESTS)))
 
 # For quicker address sanitizer build, hope that regular JSON parsing is tested enough by parallel and join tests
-fewer-tests: tippecanoe tippecanoe-decode geobuf-test raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit
+fewer-tests: tippecanoe tippecanoe-decode geobuf-test raw-tiles-test parallel-test radix-sort-test pbf-test join-test enumerate-test decode-test join-filter-test unit
 
 # XXX Use proper makefile rules instead of a for loop
 %.json.checkbuf:
@@ -143,6 +181,42 @@ fewer-tests: tippecanoe tippecanoe-decode geobuf-test raw-tiles-test parallel-te
 	cmp $@.out $(patsubst %.checkbuf,%,$@)
 	rm $@.out $@.mbtiles
 
+# The result of the sort must not depend on how the sort was performed, so instead of
+# checking the sorted output against an expected copy of it, check that sorting by radix
+# produces the same tiles as sorting in memory. --prefer-radix-sort lowers the memory
+# limit to 8K, which radix() then halves again, so the radix subdivision has to recurse
+# until each bucket is under 4K. How deeply that recurses depends on how many files the
+# machine will let us open at once, but the sorted result is the same either way, so
+# this comparison doesn't depend on that.
+#
+# What sends the sort down its rarely-taken paths is the shape of the input rather than
+# the size of it: a feature whose geometry alone is bigger than the memory limit is
+# sorted as a bucket of its own, and features that share a long run of leading index
+# bits have to be subdivided until there are no bits left. The first two inputs are
+# each one of those on purpose -- several separated features too big to sort in memory,
+# and many features at one location -- and the rest are for breadth.
+radix-sort-test: tippecanoe tippecanoe-decode
+	mkdir -p tests/radix-sort
+	perl -e 'for ($$f = 0; $$f < 8; $$f++) { print "{ \"type\": \"Feature\", \"properties\": { \"f\": $$f }, \"geometry\": { \"type\": \"LineString\", \"coordinates\": ["; for ($$i = 0; $$i < 2000; $$i++) { print "," unless $$i == 0; printf "[%f,%f]", $$f * 40 - 175 + $$i * 0.001, $$i % 2 * 0.5 - 20; } print "] } }\n"; }' > tests/radix-sort/bigfeatures.json
+	perl -e 'for ($$i = 0; $$i < 500; $$i++) { print "{ \"type\": \"Feature\", \"properties\": { \"i\": $$i }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ 17, 42 ] } }\n"; }' > tests/radix-sort/onelocation.json
+	$(MAKE) radix-sort-compare RADIXIN="tests/radix-sort/bigfeatures.json" RADIXARGS="-z4"
+	$(MAKE) radix-sort-compare RADIXIN="tests/radix-sort/onelocation.json" RADIXARGS="-z4"
+	$(MAKE) radix-sort-compare RADIXIN="tests/feature-filter/in.json" RADIXARGS="-z4"
+	$(MAKE) radix-sort-compare RADIXIN="tests/ne_110m_ocean/in.json" RADIXARGS="-z4"
+	$(MAKE) radix-sort-compare RADIXIN="tests/border/in.json" RADIXARGS="-z4"
+	$(MAKE) radix-sort-compare RADIXIN="tests/loop/in.json" RADIXARGS="-z4"
+	$(MAKE) radix-sort-compare RADIXIN="tests/tl_2022_11_tract/in.json.gz" RADIXARGS="-z4"
+	$(MAKE) radix-sort-compare RADIXIN="tests/epsg-3857/in.json" RADIXARGS="-z4 -sEPSG:3857"
+	rm -r tests/radix-sort
+
+radix-sort-compare:
+	./tippecanoe -q -f $(RADIXARGS) -o tests/radix-sort/memory.mbtiles $(RADIXIN)
+	./tippecanoe -q -f $(RADIXARGS) -aR -o tests/radix-sort/radix.mbtiles $(RADIXIN)
+	./tippecanoe-decode -x generator -x generator_options -x name -x description tests/radix-sort/memory.mbtiles > tests/radix-sort/memory.json
+	./tippecanoe-decode -x generator -x generator_options -x name -x description tests/radix-sort/radix.mbtiles > tests/radix-sort/radix.json
+	cmp tests/radix-sort/memory.json tests/radix-sort/radix.json
+	rm tests/radix-sort/memory.mbtiles tests/radix-sort/radix.mbtiles tests/radix-sort/memory.json tests/radix-sort/radix.json
+
 parallel-test: $(eval SHELL:=$(ADVSHELL))
 	mkdir -p tests/parallel
 	perl -e 'for ($$i = 0; $$i < 20; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; $$k = rand(1); $$v = rand(1); print "{ \"type\": \"Feature\", \"properties\": { \"yes\": \"no\", \"who\": 1, \"$$k\": \"$$v\" }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in1.json
@@ -366,7 +440,7 @@ overzoom-test: tippecanoe-overzoom
 	./tippecanoe-overzoom -b0 --deduplicate-by-id -o tests/pbf/merged-dedup.pbf -t 1/1/0 tests/pbf/1.json.dir/0/0/0.pbf 0/0/0 tests/pbf/2.json.dir/0/0/0.pbf 0/0/0
 	./tippecanoe-decode tests/pbf/merged-dedup.pbf 1 1 0 > tests/pbf/merged-dedup.pbf.json.check
 	cmp tests/pbf/merged-dedup.pbf.json.check tests/pbf/merged-dedup.pbf.json
-	rm -r tests/pbf/1.json.dir tests/pbf/2.json.dir tests/pbf/merged-nodedup.pbf tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-dedup.pbf.json.check
+	rm -r tests/pbf/1.json.dir tests/pbf/2.json.dir tests/pbf/merged-nodedup.pbf tests/pbf/merged-nodedup.pbf.json.check tests/pbf/merged-dedup.pbf tests/pbf/merged-dedup.pbf.json.check
 
 join-test: tippecanoe tippecanoe-decode tile-join
 	./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}'
@@ -572,6 +646,12 @@ csv-test: tippecanoe tippecanoe-decode
 	cmp tests/csv/out.mbtiles.json.check tests/csv/out.mbtiles.json
 	rm -f tests/csv/out.mbtiles.json.check tests/csv/out.mbtiles
 
+flatgeobuf-test: tippecanoe tippecanoe-decode
+	./tippecanoe -q -f -z0 -l numeric-properties -o tests/flatgeobuf/numeric-properties.mbtiles tests/flatgeobuf/numeric-properties.fgb
+	./tippecanoe-decode tests/flatgeobuf/numeric-properties.mbtiles 0 0 0 > tests/flatgeobuf/numeric-properties.json.check
+	cmp tests/flatgeobuf/numeric-properties.json.check tests/flatgeobuf/numeric-properties.json
+	rm -f tests/flatgeobuf/numeric-properties.mbtiles tests/flatgeobuf/numeric-properties.json.check
+
 layer-json-test: tippecanoe tippecanoe-decode
 	# GeoJSON with description and named layer
 	./tippecanoe -q -z0 -r1 -yNAME -f -o tests/layer-json/out.mbtiles -L'{"file":"tests/ne_110m_populated_places/in.json", "description":"World cities", "layer":"places"}'

README.md

@@ -20,11 +20,11 @@ the density and texture of the data rather than a simplification from dropping
 supposedly unimportant features or clustering or aggregating them.
 
 If you give it all of OpenStreetMap and zoom out, it should give you back
-something that looks like "[All Streets](http://benfry.com/allstreets/map5.html)"
+something that looks like "[All Streets](https://benfry.com/allstreets/)"
 rather than something that looks like an Interstate road atlas.
 
 If you give it all the building footprints in Los Angeles and zoom out
-far enough that most individual buildings are no longer discernable, you
+far enough that most individual buildings are no longer discernible, you
 should still be able to see the extent and variety of development in every neighborhood,
 not just the largest downtown buildings.
 
@@ -58,7 +58,7 @@ Usage
 -----
 
 ```
-$ tippecanoe -o file.mbtiles [options] [file.json file.json.gz file.fgb ...]
+$ tippecanoe -o file.mbtiles [options] [file.json file.json.gz file.fgb file.csv ...]
 ```
 
 If no files are specified, it reads GeoJSON from the standard input.
@@ -291,6 +291,11 @@ If your features have a lot of attributes, use `-y` to keep only the ones you re
 
 If your input is formatted as newline-delimited GeoJSON, use `-P` to make input parsing a lot faster.
 
+Many of the options below have a short form beginning with `-a` or `-p`. These are the
+`-a`_letters_ (`--additional=`_letters_) and `-p`_letters_ (`--prevent=`_letters_) options, and each
+accepts several letters at once, so for example `-ansd` is the same as `-an -as -ad`
+and `-pkC` is the same as `-pk -pC`.
+
 ### Output tileset
 
  * `-o` _file_`.mbtiles`, _file_`.pmtiles` or `--output=`_file_`.mbtiles`: Name the output file.
@@ -301,9 +306,9 @@ If your input is formatted as newline-delimited GeoJSON, use `-P` to make input
 
 ### Tileset description and attribution
 
- * `-n` _name_ or `--name=`_name_: Human-readable name for the tileset (default file.json)
+ * `-n` _name_ or `--name=`_name_: Human-readable name for the tileset (default: the name of the output file or directory)
  * `-A` _text_ or `--attribution=`_text_: Attribution (HTML) to be shown with maps that use data from this tileset.
- * `-N` _description_ or `--description=`_description_: Description for the tileset (default file.mbtiles)
+ * `-N` _description_ or `--description=`_description_: Description for the tileset (default: the name of the output file or directory)
 
 ### Input files and layer names
 
@@ -320,7 +325,7 @@ If your input is formatted as newline-delimited GeoJSON, use `-P` to make input
 tippecanoe -z5 -o world.mbtiles -L'{"file":"ne_10m_admin_0_countries.json", "layer":"countries", "description":"Natural Earth countries"}'
 ```
 
-CSV input files currently support only Point geometries, from columns named `latitude`, `longitude`, `lat`, `lon`, `long`, `lng`, `x`, or `y`.
+CSV input files currently support only Point geometries, from columns named `lat`, `lon`, `long`, `lng`, `x`, or `y`, or from any column whose name contains `latitude` or `longitude`. Column names are matched without regard to case.
 
 ### Parallel processing of input
 
@@ -352,7 +357,7 @@ Parallel processing will also be automatic if the input file is in FlatGeobuf fo
    specified maximum zoom and to any levels added beyond that.
  * `--extend-zooms-if-still-dropping-maximum=`_count_: Increase the maxzoom if features are still being dropped at that zoom level
    by up to _count_ zoom levels.
- * `-at` or `--generate-variable-depth-tile-pyramid`: Don't produce child tiles for any tile that should be sufficient to be overzoomed to any higher zoom level. Such tiles will be produced with maximum detail and no simplification or polygon cleaning. Tiles with point features below the basezoom or where any features have to be dropped dynamically, or which contain too many features or bytes with full detail, will be written out with normal detail and split into child tiles. Tilesets generated with this option are suitable for use only with tile servers that will find the appropriate tile to overzoom from and will simplify and clean the geometries appropriately before serving the tile.
+ * `-at` or `--generate-variable-depth-tile-pyramid`: Don't produce child tiles for any tile that should be sufficient to be overzoomed to any higher zoom level. Such tiles will be produced with maximum detail, retaining every vertex that is not collinear with its neighbors, but their polygons are still cleaned so that overlapping areas are merged rather than left stacked. Tiles with point features below the basezoom, or which contain too many features or bytes with full detail, will be written out with normal detail and split into child tiles. If any features have to be dropped dynamically at a zoom level, no tile in that zoom level is truncated, so that the whole zoom level is reduced consistently; any tile that was already being skipped because an ancestor had been truncated is written out after all. Tilesets generated with this option are suitable for use only with tile servers that will find the appropriate tile to overzoom from and will simplify the geometries appropriately before serving the tile.
  * `-R` _zoom_`/`_x_`/`_y_ or `--one-tile=`_zoom_`/`_x_`/`_y_: Set the minzoom and maxzoom to _zoom_ and produce only
    the single specified tile at that zoom level.
 
@@ -415,8 +420,9 @@ be reduced to the maximum that can be used with the specified _maxzoom_.
  * `-Y`_attribute_`:`_description_ or `--attribute-description=`_attribute_`:`_description_: Set the `description` for the specified attribute in the tileset metadata to _description_ instead of the usual `String`, `Number`, or `Boolean`.
  * `-E`_attribute_`:`_operation_ or `--accumulate-attribute=`_attribute_`:`_operation_: Preserve the named _attribute_ from features
    that are dropped, coalesced-as-needed, or clustered. The _operation_ may be
-   `sum`, `product`, `mean`, `max`, `min`, `concat`, or `comma`
+   `sum`, `product`, `mean`, `max`, `min`, `concat`, `comma`, or `count`
    to specify how the named _attribute_ is accumulated onto the attribute of the same name in a feature that does survive.
+   (The `count` operation replaces the attribute with the number of features, including the survivor, that carried it.)
    The attributes and operations may also be specified as JSON keys and values: `--accumulate-attribute='{"attr": "operation", "attr2": "operation2"}'`.
  * `--set-attribute` _attribute_`:`_value_: Set the value of the specified _attribute_ in each feature to the specified _value_. This is mostly useful to give an attribute in each feature an initial value for `--accumulate-attribute`.
    The attributes and values may also be specified as JSON keys and values: `--set-attribute='{"attr": value, "attr2": value}'`.
@@ -466,16 +472,17 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
    If you use `-Bg`, it will guess a zoom level that will keep at most 50,000 features in the densest tile.
    You can also specify a marker-width with `-Bg`*width* to allow fewer features in the densest tile to
    compensate for the larger marker, or `-Bf`*number* to allow at most *number* features in the densest tile.
- * `--retain-points-multiplier=`_multiple_: Retain the specified multiple of points instead of just the number of points that would ordinarily be retained by the drop rate. These can be thinned out later with the `-m` option to `tippecanoe-overzoom`. The start of each cluster is marked in the feature sequence by the `tippecanoe:retain_points_multiplier_first` attribute. The `--tile-size-limit` will also be extended at low zoom levels to allow for the multiplied features.
+ * `--retain-points-multiplier=`_multiple_: Retain the specified multiple of points instead of just the number of points that would ordinarily be retained by the drop rate. These can be thinned out later with the `-m` option to `tippecanoe-overzoom`. The start of each cluster is marked in the feature sequence by the `tippecanoe:retain_points_multiplier_first` attribute. The maximum tile size (`--maximum-tile-bytes`) will also be extended at low zoom levels to allow for the multiplied features.
  * `--drop-denser=`_percentage_: When dropping dots at zoom levels below the base zoom, give the specified _percentage_
    preference to retaining points in sparse areas and dropping points in dense areas.
- * `--limit-base-zoom-to-maximum-zoom` or `-Pb`: Limit the guessed base zoom not to exceed the maxzoom, even if this would put more than the requested number of features in a base zoom tile.
+ * `--limit-base-zoom-to-maximum-zoom` or `-pb`: Limit the guessed base zoom not to exceed the maxzoom, even if this would put more than the requested number of features in a base zoom tile.
  * `-al` or `--drop-lines`: Let "dot" dropping at lower zooms apply to lines too
  * `-ap` or `--drop-polygons`: Let "dot" dropping at lower zooms apply to polygons too
  * `-K` _distance_ or `--cluster-distance=`_distance_: Cluster points (as with `--cluster-densest-as-needed`, but without the experimental discovery process) that are approximately within _distance_ of each other. The units are tile coordinates within a nominally 256-pixel tile, so the maximum value of 255 allows only one feature per tile. Values around 10 are probably appropriate for typical marker sizes. See `--cluster-densest-as-needed` below for behavior.
  * `-k` _zoom_ or `--cluster-maxzoom=`_zoom_: Max zoom on which to cluster points if clustering is enabled.
  * `-kg` or `--cluster-maxzoom=g`: Set `--cluster-maxzoom=` to `maxzoom - 1` so that all features are visible at the maximum zoom level.
  * `--preserve-point-density-threshold=`_level_: At the low zoom levels, do not reduce point density below the specified _level_, even if the specified drop rate would normally call for it, so that low-density areas of the map do not appear blank. The unit is the distance between preserved points, as a fraction of the size of a tile. Values of 32 or 64 are probably appropriate for typical marker sizes.
+ * `--preserve-multiplier-density-threshold=`_level_: As with `--preserve-point-density-threshold`, but for the additional features retained by `--retain-points-multiplier`: features that would otherwise be dropped are instead added to the multiplier cluster if they are farther than the specified _level_ from the previous retained feature, so that sparse areas still have features available to be thinned to. The unit is the same as for `--preserve-point-density-threshold`.
 
 ### Dropping a fraction of features to keep under tile size limits
 
@@ -484,10 +491,11 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
  * `-an` or `--drop-smallest-as-needed`: Dynamically drop the smallest features (physically smallest: the shortest lines or the smallest polygons) from each zoom level to keep large tiles under the 500K size limit.
  * `--drop-by-attribute-as-needed=`_attribute_: Dynamically drop features with the lowest values of the specified numeric _attribute_ from each zoom level to keep large tiles under the 500K size limit. Use `--drop-by-attribute-order=desc` to instead drop features with the highest values.
  * `-aN` or `--coalesce-smallest-as-needed`: Dynamically combine the smallest features (physically smallest: the shortest lines or the smallest polygons or the densest points) from each zoom level into other nearby features to keep large tiles under the 500K size limit. This option will probably not help very much with LineStrings. It is mostly intended for polygons, to maintain the full original area covered by polygons while still reducing the feature count somehow. The attributes of the small polygons are *not* preserved into the combined features (except through `--accumulate-attribute`), only their geometry. Furthermore, the polygons to which nested polygons are coalesced may not necessarily be the immediately enclosing features.
- * `-aD` or `--coalesce-densest-as-needed`: Dynamically combine the densest features from each zoom level into other nearby features to keep large tiles under the 500K size limit. (Again, mostly useful for polygons.)
+ * `-aS` or `--coalesce-densest-as-needed`: Dynamically combine the densest features from each zoom level into other nearby features to keep large tiles under the 500K size limit. (Again, mostly useful for polygons.)
- * `-aS` or `--coalesce-fraction-as-needed`: Dynamically combine a fraction of features from each zoom level into other nearby features to keep large tiles under the 500K size limit. (Again, mostly useful for polygons.)
+ * `-aD` or `--coalesce-fraction-as-needed`: Dynamically combine a fraction of features from each zoom level into other nearby features to keep large tiles under the 500K size limit. (Again, mostly useful for polygons.)
  * `-pd` or `--force-feature-limit`: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries. (This is like `-ad` but applies to each tile individually, not to the entire zoom level.) You probably don't want to use this.
- * `-aC` or `--cluster-densest-as-needed`: If a tile is too large, try to reduce its size by increasing the minimum spacing between features, and leaving one placeholder feature from each group.  The remaining feature will be given a `"clustered": true` attribute to indicate that it represents a cluster, a `"point_count"` attribute to indicate the number of features that were clustered into it, and a `"sqrt_point_count"` attribute to indicate the relative width of a feature to represent the cluster. If the features being clustered are points, the representative feature will be located at the average of the original points' locations; otherwise, one of the original features will be left as the representative.
+ * `-aC` or `--cluster-densest-as-needed`: If a tile is too large, try to reduce its size by increasing the minimum spacing between features, and leaving one placeholder feature from each group.  The remaining feature will be given a `"clustered": true` attribute to indicate that it represents a cluster, a `"point_count"` attribute to indicate the number of features that were clustered into it, a `"point_count_abbreviated"` attribute containing that count abbreviated for display (for example `1.2k` or `15k`), and a `"sqrt_point_count"` attribute to indicate the relative width of a feature to represent the cluster. If the features being clustered are points, the representative feature will be located at the average of the original points' locations (unless you use `--keep-point-cluster-position`); otherwise, one of the original features will be left as the representative.
+ * `-aa` or `--keep-point-cluster-position`: When clustering points, leave the representative feature at the location of the first point of the cluster instead of moving it to the average of the clustered points' locations.
 
 ### Dropping tightly overlapping features
 
@@ -509,7 +517,7 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
 
 ### Attempts to improve shared polygon boundaries
 
- * `-ab` or `--detect-shared-borders`: DEPRECATED. In the manner of [TopoJSON](https://github.com/mbostock/topojson/wiki/Introduction), detect borders that are shared between multiple polygons and simplify them identically in each polygon. This takes more time and memory than considering each polygon individually. Use `no-simplification-of-shared-nodes` instead, which is faster and more correct.
+ * `-ab` or `--detect-shared-borders`: DEPRECATED. In the manner of [TopoJSON](https://github.com/mbostock/topojson/wiki/Introduction), detect borders that are shared between multiple polygons and simplify them identically in each polygon. This takes more time and memory than considering each polygon individually. Use `--no-simplification-of-shared-nodes` instead, which is faster and more correct.
  * `-aL` or `--grid-low-zooms`: At all zoom levels below _maxzoom_, snap all lines and polygons to a stairstep grid instead of allowing diagonals. You will also want to specify a tile resolution, probably `-D8`. This option provides a way to display continuous parcel, gridded, or binned data at low zooms without overwhelming the tiles with tiny polygons, since features will either get stretched out to the grid unit or lost entirely, depending on how they happened to be aligned in the original data. You probably don't want to use this.
 
 ### Controlling clipping to tile boundaries
@@ -568,9 +576,16 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
  * `-q` or `--quiet`: Work quietly instead of reporting progress or warning messages
  * `-Q` or `--no-progress-indicator`: Don't report progress, but still give warnings
  * `-U` _seconds_ or `--progress-interval=`_seconds_: Don't report progress more often than the specified number of _seconds_.
- * `-u` or `--json-progress`: like `-quiet` but logs progress as a JSON object. Use in combination with `-U`.
+ * `-u` or `--json-progress`: like `--quiet` but logs progress as a JSON object. Use in combination with `-U`.
+
+### Version
+
  * `-v` or `--version`: Report Tippecanoe's version number
 
+### Help
+
+ * `-H` or `--help`: List the available options and exit
+
 ### Filters
 
  * `-C` _command_ or `--prefilter=`_command_: Specify a shell filter command to be run at the start of assembling each tile
@@ -641,7 +656,7 @@ If you have a feature like this:
 }
 ```
 
-with a `tippecanoe` object specifiying a `maxzoom` of 9 and a `minzoom` of 4, the feature
+with a `tippecanoe` object specifying a `maxzoom` of 9 and a `minzoom` of 4, the feature
 will only appear in the vector tiles for zoom levels 4 through 9. Note that the `tippecanoe`
 object belongs to the Feature, not to its `properties`. If you specify a `minzoom` for a feature,
 it will be preserved down to that zoom level even if dot-dropping with `-r` would otherwise have
@@ -674,11 +689,11 @@ Geometric simplifications
 At every zoom level, line and polygon features are subjected to Douglas-Peucker
 simplification to the resolution of the tile.
 
-For point features, it drops 1/2.5 of the dots for each zoom level above the
+For point features, it keeps only 1/2.5 of the dots for each zoom level below the
 point base zoom (which is normally the same as the `-z` max zoom, but can be
 a different zoom specified with `-B` if you have precise but sparse data).
 I don't know why 2.5 is the appropriate number, but the densities of many different
-data sets fall off at about this same rate. You can use -r to specify a different rate.
+data sets fall off at about this same rate. You can use `-r` to specify a different rate.
 
 You can use the gamma option to thin out especially dense clusters of points.
 For any area where dots are closer than one pixel together (at whatever zoom level),
@@ -704,8 +719,13 @@ lower resolutions before failing if it still doesn't fit.
 Development
 -----------
 
-Requires sqlite3 and zlib (should already be installed on MacOS). Rebuilding the manpage
+Requires sqlite3 and zlib (should already be installed on MacOS).
-uses md2man (`gem install md2man`).
+
+The manpage is generated from this README by `make docs`, which uses
+[go-md2man](https://github.com/cpuguy83/go-md2man) (`brew install go-md2man` or
+`apt-get install go-md2man`). You don't have to run it yourself: CI regenerates the
+manpage and fails if the committed copy doesn't match, so it will tell you if an
+edit here needs `make docs` run against it.
 
 Linux:
 
@@ -719,15 +739,16 @@ and perhaps
 
     make install
 
-Tippecanoe now requires features from the 2011 C++ standard. If your compiler is older than
+Tippecanoe requires features from the 2017 C++ standard (it is built with `-std=c++17`).
-that, you will need to install a newer one. On MacOS, updating to the latest XCode should
+If your compiler is older than that, you will need to install a newer one. On MacOS, updating
-get you a new enough version of `clang++`. On Linux, you should be able to upgrade `g++` with
+to the latest XCode should get you a new enough version of `clang++`. On Linux, you should be
+able to upgrade `g++` with
 
 ```sh
 sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
 sudo apt-get update -y
-sudo apt-get install -y g++-5
+sudo apt-get install -y g++-9
-export CXX=g++-5
+export CXX=g++-9
 ```
 
 Docker Image
@@ -780,11 +801,14 @@ The options are:
  * `-o` *out.mbtiles*, *out.pmtiles* or `--output=`*out.mbtiles*: Write the new tiles to the specified .mbtiles file.
  * `-e` *directory* or `--output-to-directory=`*directory*: Write the new tiles to the specified directory instead of to an mbtiles file.
  * `-f` or `--force`: Remove *out.mbtiles* if it already exists.
- * `-r` or `--read-from`: list of input mbtiles to read from.
+
+### Input tilesets
+
+ * `-r` *file* or `--read-from=`*file*: Read the list of input tilesets from the named *file*, one filename per line, instead of naming them all on the command line.
 
 ### Overzooming
 
- * `--overzoom`: If one of the source tilesets has a larger maxzoom than the others, scale up tiles from the tilesets with the lower maxzooms so they will all have the same maxzoom in the output tileset.
+ * `-O` or `--overzoom`: If one of the source tilesets has a larger maxzoom than the others, scale up tiles from the tilesets with the lower maxzooms so they will all have the same maxzoom in the output tileset.
  * `--buffer=`_pixels_ or `-b` _pixels_: Set the size of the tile buffer in the overzoomed tiles.
 
 ### Tileset description and attribution
@@ -813,6 +837,8 @@ The options are:
  * `-x` *key* or `--exclude=`*key*: Remove attributes named *key* from the output. You can use this to remove the field you are matching against if you no longer need it after joining, or to remove any other attributes you don't want. You can use multiple `-x` options to remove multiple attributes.
  * `-X` or `--exclude-all`: Remove all attributes from the output.
  * `-y` *key* or `--include=`*key*: Remove all attributes except for those named *key* from the output. You can use multiple `-y` options to retain multiple attributes.
+ * `--exclude-all-tile-attributes`: Remove the attributes that were already present in the source tiles, keeping only any that are joined from a CSV with `-c`.
+ * `--exclude-all-tile-geometries`: Copy the features' attributes but not their geometries, so the output tiles describe the features without locating them.
  * `-i` or `--if-matched`: Only include features that matched the CSV.
  * `-j` *filter* or `--feature-filter`=*filter*: Check features against a per-layer filter (as defined in the [Mapbox GL Style Specification](https://docs.mapbox.com/mapbox-gl-js/style-spec/#other-filter)) and only include those that match. Any features in layers that have no filter specified will be passed through. Filters for the layer `"*"` apply to all layers.
  * `-J` *filter-file* or `--feature-filter-file`=*filter-file*: Like `-j`, but read the filter from a file.
@@ -827,6 +853,10 @@ The options are:
  * `--tile-stats-sample-values-limit=`*count*: Calculate `tilestats` attribute statistics based on *count* values instead of the default 1000.
  * `--tile-stats-values-limit=`*count*: Report *count* unique attribute values in `tilestats` instead of the default 100.
 
+### Progress indicator
+
+ * `-q` or `--quiet`: Work quietly instead of reporting progress
+
 Because tile-join just copies the geometries to the new .mbtiles without processing them
 (except to rescale the extents if necessary),
 it doesn't have any of tippecanoe's recourses if the new tiles are bigger than the 500K tile limit.
@@ -917,8 +947,10 @@ resolutions.
  * `-c` or `--tag-layer-and-zoom`: Include each feature's layer and zoom level as part of its `tippecanoe` object rather than as a FeatureCollection wrapper
  * `-S` or `--stats`: Just report statistics about each tile's size and the number of features in it, as a JSON structure.
  * `-f` or `--force`: Decode tiles even if polygon ring order or closure problems are detected
- * `-I` or `--integer`: Report coordinates in integer tile coordinates
+ * `-I` or `--integer-coordinates`: Report coordinates in integer tile coordinates
- * `-F` or `--fraction`: Report coordinates as a fraction of the tile extent
+ * `-F` or `--fractional-coordinates`: Report coordinates as a fraction of the tile extent
+ * `-y` _attribute_ or `--include=`*attribute*: Include only the named attributes in the decoded features, excluding all those not explicitly named. (Multiple `-y` options can be specified.)
+ * `-x` _name_ or `--exclude-metadata-row=`*name*: Omit the named row from the tileset metadata in the output. (Multiple `-x` options can be specified.)
 
 tippecanoe-json-tool
 ====================
@@ -1005,10 +1037,19 @@ and produces tile `outz/outx/outy` of `out.mvt.gz` from them.
 
 ### Options
 
- * `-b` *buffer*: Set the tile buffer in the output tile (default 5)
+ * `-o` *file* or `--output=`*file*: Write the output tile to the named *file*.
- * `-d` *detail*: Set the detail of the output tile (default 12)
+ * `-t` _zoom_`/`_x_`/`_y_ or `--source-tile=`_zoom_`/`_x_`/`_y_: Specify the coordinates of the tile to produce, so that several input tiles, each followed by its own _zoom_`/`_x_`/`_y_, can be combined into it, as in the second example above.
- * `-y` *attribute*: Retain the specified *attribute* in the output features. All attributes that are not named in a `-y` option will be removed.
+ * `-b` *buffer* or `--buffer=`*buffer*: Set the tile buffer in the output tile (default 5)
- * `-j` *filter*: Filter features using the same expression syntax as in tippecanoe.
+ * `-d` *detail* or `--full-detail=`*detail*: Set the detail of the output tile (default 12)
- * `-m`: If a tile was created with the `--retain-points-multiplier` option, thin the tile back down to its normal feature count during overzooming. The first feature from each cluster will be retained, unless `-j` is used to specify a filter, in which case the first matching filter from each cluster will be retained instead.
+ * `-y` *attribute* or `--include=`*attribute*: Retain the specified *attribute* in the output features. All attributes that are not named in a `-y` option will be removed.
+ * `-x` *attribute* or `--exclude=`*attribute*: Remove the specified *attribute* from the output features.
+ * `--exclude-prefix=`*prefix*: Remove any attribute whose name begins with the specified *prefix* from the output features.
+ * `-j` *filter* or `--feature-filter=`*filter*: Filter features using the same expression syntax as in tippecanoe.
+ * `-J` *filter-file* or `--feature-filter-file=`*filter-file*: Like `-j`, but read the filter from a file.
+ * `-m` or `--filter-points-multiplier`: If a tile was created with the `--retain-points-multiplier` option, thin the tile back down to its normal feature count during overzooming. The first feature from each cluster will be retained, unless `-j` is used to specify a filter, in which case the first matching filter from each cluster will be retained instead.
+ * `-S` *scale* or `--line-simplification=`*scale*: Simplify lines and polygons in the output tile, multiplying the standard tolerance by *scale*. The default of 0 means not to simplify at all.
+ * `--tiny-polygon-size=`*size*: Combine the area of very small polygons into small squares of the specified *size* that represent their combined area, as `tippecanoe` does. The default of 0 means not to do tiny polygon reduction at all.
+ * `--deduplicate-by-id`: When several input tiles are combined, include only the first feature with any given feature ID within each layer, so that features that appear in more than one input tile are not duplicated in the output.
  * `--preserve-input-order`: Restore a set of filtered features to its original input order
- * `--accumulate-attribute`: Behaves as in `tippecanoe` to sum attributes from the features of a multiplier cluster that are not included in the final output. The attributes from features that are filtered away with `-j` are *not* accumulated onto the output feature.
+ * `-E` *attribute*`:`*operation* or `--accumulate-attribute=`*attribute*`:`*operation*: Behaves as in `tippecanoe` to sum attributes from the features of a multiplier cluster that are not included in the final output. The attributes from features that are filtered away with `-j` are *not* accumulated onto the output feature.
+ * `--no-tile-compression`: Don't compress the PBF vector tile data in the output tile.

decode.cpp

@@ -24,6 +24,7 @@
 #include "dirtiles.hpp"
 #include "pmtiles_file.hpp"
 #include "errors.hpp"
+#include "usage.hpp"
 
 int minzoom = 0;
 int maxzoom = 32;
@@ -549,8 +550,42 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 	}
 }
 
+static const struct option long_options[] = {
+	{"Tiles to decode", 0, 0, 0},
+	{"minimum-zoom", required_argument, 0, 'Z'},
+	{"maximum-zoom", required_argument, 0, 'z'},
+	{"layer", required_argument, 0, 'l'},
+
+	{"Output format", 0, 0, 0},
+	{"projection", required_argument, 0, 's'},
+	{"fractional-coordinates", no_argument, 0, 'F'},
+	{"integer-coordinates", no_argument, 0, 'I'},
+	{"tag-layer-and-zoom", no_argument, 0, 'c'},
+	{"stats", no_argument, 0, 'S'},
+
+	{"Filtering the output", 0, 0, 0},
+	{"include", required_argument, 0, 'y'},
+	{"exclude-metadata-row", required_argument, 0, 'x'},
+
+	{"Ignoring errors in the input", 0, 0, 0},
+	{"force", no_argument, 0, 'f'},
+
+	{0, 0, 0, 0},
+};
+
+// the options above, with the usage message headings removed
+static struct option real_long_options[sizeof(long_options) / sizeof(long_options[0])];
+
 void usage(char **argv) {
-	fprintf(stderr, "Usage: %s [-s projection] [-Z minzoom] [-z maxzoom] [-l layer ...] file.mbtiles [zoom x y]\n", argv[0]);
+	static const char *const forms[] = {
+		"[options] tileset",
+		"[options] tileset zoom x y",
+		NULL,
+	};
+
+	print_usage(stderr, argv[0], forms, long_options, NULL);
+	fprintf(stderr, "\nThe tileset can be an .mbtiles or .pmtiles file or a directory of tiles,\n");
+	fprintf(stderr, "or, if zoom/x/y is specified, a single .pbf tile.\n");
 	exit(EXIT_ARGS);
 }
 
@@ -564,33 +599,10 @@ int main(int argc, char **argv) {
 	std::set<std::string> exclude_meta;
 	int coordinate_mode = 0;
 
-	struct option long_options[] = {
+	strip_usage_headings(long_options, real_long_options);
-		{"projection", required_argument, 0, 's'},
+	std::string getopt_str = getopt_string(real_long_options);
-		{"fractional-coordinates", no_argument, 0, 'F'},
-		{"integer-coordinates", no_argument, 0, 'I'},
-		{"maximum-zoom", required_argument, 0, 'z'},
-		{"minimum-zoom", required_argument, 0, 'Z'},
-		{"layer", required_argument, 0, 'l'},
-		{"tag-layer-and-zoom", no_argument, 0, 'c'},
-		{"stats", no_argument, 0, 'S'},
-		{"force", no_argument, 0, 'f'},
-		{"exclude-metadata-row", required_argument, 0, 'x'},
-		{"include", required_argument, 0, 'y'},
-		{0, 0, 0, 0},
-	};
 
-	std::string getopt_str;
+	while ((i = getopt_long(argc, argv, getopt_str.c_str(), real_long_options, NULL)) != -1) {
-	for (size_t lo = 0; long_options[lo].name != NULL; lo++) {
-		if (long_options[lo].val > ' ') {
-			getopt_str.push_back(long_options[lo].val);
-
-			if (long_options[lo].has_arg == required_argument) {
-				getopt_str.push_back(':');
-			}
-		}
-	}
-
-	while ((i = getopt_long(argc, argv, getopt_str.c_str(), long_options, NULL)) != -1) {
 		switch (i) {
 		case 0:
 			break;

enumerate.cpp

@@ -1,8 +1,11 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
+#include <getopt.h>
+#include <string>
 #include <sqlite3.h>
 #include "errors.hpp"
+#include "usage.hpp"
 
 void enumerate(char *fname) {
 	sqlite3 *db;
@@ -48,8 +51,20 @@ void enumerate(char *fname) {
 	}
 }
 
+// there are no options, but the table is still what the usage message
+// and the getopt string are derived from, so that they will keep up
+// with any options that are added later
+static const struct option long_options[] = {
+	{0, 0, 0, 0},
+};
+
 void usage(char **argv) {
-	fprintf(stderr, "Usage: %s file.mbtiles ...\n", argv[0]);
+	static const char *const forms[] = {
+		"file.mbtiles ...",
+		NULL,
+	};
+
+	print_usage(stderr, argv[0], forms, long_options, NULL);
 	exit(EXIT_ARGS);
 }
 
@@ -58,7 +73,9 @@ int main(int argc, char **argv) {
 	// extern char *optarg;
 	int i;
 
-	while ((i = getopt(argc, argv, "")) != -1) {
+	std::string getopt_str = getopt_string(long_options);
+
+	while ((i = getopt_long(argc, argv, getopt_str.c_str(), long_options, NULL)) != -1) {
 		usage(argv);
 	}
 

flatgeobuf.cpp

@@ -163,13 +163,13 @@ void readFeature(const FlatGeobuf::Feature *feature, long long feature_sequence_
 
 		serial_val sv;
 		if (col_type == FlatGeobuf::ColumnType_Byte) {
-			sv.type = mvt_sint;
+			sv.type = mvt_double;
 			int8_t byte_val;
 			memcpy(&byte_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(byte_val));
 			sv.s = std::to_string(byte_val);
 			p_pos += sizeof(uint16_t) + sizeof(byte_val);
 		} else if (col_type == FlatGeobuf::ColumnType_UByte) {
-			sv.type = mvt_uint;
+			sv.type = mvt_double;
 			uint8_t ubyte_val;
 			memcpy(&ubyte_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(ubyte_val));
 			sv.s = std::to_string(ubyte_val);
@@ -185,43 +185,43 @@ void readFeature(const FlatGeobuf::Feature *feature, long long feature_sequence_
 			}
 			p_pos += sizeof(uint16_t) + sizeof(bool_val);
 		} else if (col_type == FlatGeobuf::ColumnType_Short) {
-			sv.type = mvt_sint;
+			sv.type = mvt_double;
 			int16_t short_val;
 			memcpy(&short_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(short_val));
 			sv.s = std::to_string(short_val);
 			p_pos += sizeof(uint16_t) + sizeof(short_val);
 		} else if (col_type == FlatGeobuf::ColumnType_UShort) {
-			sv.type = mvt_uint;
+			sv.type = mvt_double;
 			uint16_t ushort_val;
 			memcpy(&ushort_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(ushort_val));
 			sv.s = std::to_string(ushort_val);
 			p_pos += sizeof(uint16_t) + sizeof(ushort_val);
 		} else if (col_type == FlatGeobuf::ColumnType_Int) {
-			sv.type = mvt_sint;
+			sv.type = mvt_double;
 			int32_t int_val;
 			memcpy(&int_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(int_val));
 			sv.s = std::to_string(int_val);
 			p_pos += sizeof(uint16_t) + sizeof(int_val);
 		} else if (col_type == FlatGeobuf::ColumnType_UInt) {
-			sv.type = mvt_uint;
+			sv.type = mvt_double;
 			uint32_t uint_val;
 			memcpy(&uint_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(uint_val));
 			sv.s = std::to_string(uint_val);
 			p_pos += sizeof(uint16_t) + sizeof(uint_val);
 		} else if (col_type == FlatGeobuf::ColumnType_Long) {
-			sv.type = mvt_sint;
+			sv.type = mvt_double;
 			int64_t long_val;
 			memcpy(&long_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(long_val));
 			sv.s = std::to_string(long_val);
 			p_pos += sizeof(uint16_t) + sizeof(long_val);
 		} else if (col_type == FlatGeobuf::ColumnType_ULong) {
-			sv.type = mvt_uint;
+			sv.type = mvt_double;
-			int64_t ulong_val;
+			uint64_t ulong_val;
 			memcpy(&ulong_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(ulong_val));
 			sv.s = std::to_string(ulong_val);
 			p_pos += sizeof(uint16_t) + sizeof(ulong_val);
 		} else if (col_type == FlatGeobuf::ColumnType_Float) {
-			sv.type = mvt_float;
+			sv.type = mvt_double;
 			float float_val;
 			memcpy(&float_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(float_val));
 			sv.s = milo::dtoa_milo(float_val);

jsontool.cpp

@@ -13,6 +13,7 @@
 #include "geojson-loop.hpp"
 #include "milo/dtoa_milo.h"
 #include "errors.hpp"
+#include "usage.hpp"
 
 int fail = EXIT_SUCCESS;
 bool wrap = false;
@@ -407,34 +408,45 @@ void process(FILE *fp, const char *fname) {
 	json_end(jp);
 }
 
+static const struct option long_options[] = {
+	{"Wrapping the output", 0, 0, 0},
+	{"wrap", no_argument, 0, 'w'},
+
+	{"Sorting and joining", 0, 0, 0},
+	{"extract", required_argument, 0, 'e'},
+	{"csv", required_argument, 0, 'c'},
+	{"empty-csv-columns-are-null", no_argument, &pe, 1},
+
+	{"", 0, 0, 0},
+	{"prevent", required_argument, 0, 'p'},
+
+	{0, 0, 0, 0},
+};
+
+// the options above, with the usage message headings removed
+static struct option real_long_options[sizeof(long_options) / sizeof(long_options[0])];
+
+void usage(char **argv) {
+	static const char *const forms[] = {
+		"[options] [file.json ...]",
+		NULL,
+	};
+
+	print_usage(stderr, argv[0], forms, long_options, NULL);
+	fprintf(stderr, "\nIf no files are named, the JSON is read from the standard input.\n");
+	exit(EXIT_ARGS);
+}
+
 int main(int argc, char **argv) {
 	const char *csv = NULL;
 
-	struct option long_options[] = {
+	strip_usage_headings(long_options, real_long_options);
-		{"wrap", no_argument, 0, 'w'},
+	std::string getopt_str = getopt_string(real_long_options);
-		{"extract", required_argument, 0, 'e'},
-		{"csv", required_argument, 0, 'c'},
-		{"empty-csv-columns-are-null", no_argument, &pe, 1},
-		{"prevent", required_argument, 0, 'p'},
-
-		{0, 0, 0, 0},
-	};
-
-	std::string getopt_str;
-	for (size_t lo = 0; long_options[lo].name != NULL; lo++) {
-		if (long_options[lo].val > ' ') {
-			getopt_str.push_back(long_options[lo].val);
-
-			if (long_options[lo].has_arg == required_argument) {
-				getopt_str.push_back(':');
-			}
-		}
-	}
 
 	extern int optind;
 	int i;
 
-	while ((i = getopt_long(argc, argv, getopt_str.c_str(), long_options, NULL)) != -1) {
+	while ((i = getopt_long(argc, argv, getopt_str.c_str(), real_long_options, NULL)) != -1) {
 		switch (i) {
 		case 0:
 			break;
@@ -461,8 +473,7 @@ int main(int argc, char **argv) {
 			break;
 
 		default:
-			fprintf(stderr, "Unexpected option -%c\n", i);
+			usage(argv);
-			exit(EXIT_ARGS);
 		}
 	}
 

mapbox/geometry/wagyu/topology_correction.hpp

@@ -1296,7 +1296,10 @@ void correct_tree(ring_manager<T>& manager) {
         }
         if (!found) {
             if ((*itr)->is_hole()) {
-                throw std::runtime_error("Could not properly place hole to a parent.");
+                // No remaining ring contains this hole (degenerate input, e.g. stacked
+                // duplicate rings from coalesced tiny-polygon placeholders); drop it
+                // rather than fail the entire tiling run.
+                remove_ring_and_points(*itr, manager, false);
             } else {
                 // Assign to base of tree by passing nullptr
                 reassign_as_child(*itr, static_cast<ring_ptr<T>>(nullptr), manager);

mbtiles.cpp

@@ -701,7 +701,10 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
 	m.strategies_json = stringify_strategies(strategies);
 
 	if (std::isinf(droprate)) {
-		droprate = LLONG_MAX;
+		// JSON has no representation for infinity, so substitute a huge
+		// finite value. The cast is explicit because LLONG_MAX itself is
+		// not representable as a double and rounds up to 2^63.
+		droprate = (double) LLONG_MAX;
 	}
 	if (basezoom != maxzoom || droprate != 2.5 || retain_points_multiplier != 1) {
 		m.decisions_json = std::string("{") +

mvt.hpp

@@ -93,12 +93,21 @@ struct mvt_value {
 		long long sint_value;
 		bool bool_value;
 		int null_value;
+		// Initializing string_value initializes the union's full width, which
+		// the static_assert below checks. Setting only a narrower member (a
+		// double, say) would leave the remaining bytes indeterminate, and the
+		// implicit copy constructor copies the union as a whole, so those
+		// bytes get read even when they aren't the member in use.
 		struct {
 			size_t off;
 			size_t len;
-		} string_value;
+		} string_value = {0, 0};
 	} numeric_value;
 
+	static_assert(sizeof(numeric_value) == sizeof(numeric_value.string_value),
+		      "string_value must span the whole union, since its default member "
+		      "initializer is what initializes the union");
+
 	std::string get_string_value() const {
 		if (type == mvt_string) {
 			return std::string(*s, numeric_value.string_value.off, numeric_value.string_value.len);

overzoom.cpp

@@ -11,6 +11,7 @@
 #include "text.hpp"
 #include "read_json.hpp"
 #include "projection.hpp"
+#include "usage.hpp"
 
 extern char *optarg;
 extern int optind;
@@ -30,11 +31,60 @@ std::set<std::string> keep;
 std::set<std::string> exclude;
 std::vector<std::string> exclude_prefix;
 
+static const struct option long_options[] = {
+	{"Output tile", 0, 0, 0},
+	{"output", required_argument, 0, 'o'},
+	{"source-tile", required_argument, 0, 't'},
+	{"no-tile-compression", no_argument, 0, 'd' & 0x1F},
+
+	{"Tile resolution", 0, 0, 0},
+	{"full-detail", required_argument, 0, 'd'},
+	{"buffer", required_argument, 0, 'b'},
+
+	{"Filtering feature attributes", 0, 0, 0},
+	{"include", required_argument, 0, 'y'},
+	{"exclude", required_argument, 0, 'x'},
+	{"exclude-prefix", required_argument, 0, 'x' & 0x1F},
+
+	{"Modifying feature attributes", 0, 0, 0},
+	{"accumulate-attribute", required_argument, 0, 'E'},
+
+	{"Filtering features", 0, 0, 0},
+	{"feature-filter", required_argument, 0, 'j'},
+	{"feature-filter-file", required_argument, 0, 'J'},
+	{"filter-points-multiplier", no_argument, 0, 'm'},
+	{"deduplicate-by-id", no_argument, 0, 'i' & 0x1F},
+
+	{"Line and polygon simplification", 0, 0, 0},
+	{"line-simplification", required_argument, 0, 'S'},
+	{"tiny-polygon-size", required_argument, 0, 's' & 0x1F},
+
+	{"Reordering features within the tile", 0, 0, 0},
+	{"preserve-input-order", no_argument, 0, 'o' & 0x1F},
+
+	{"", 0, 0, 0},
+	{"unidecode-data", required_argument, 0, 'u' & 0x1F},
+
+	{0, 0, 0, 0},
+};
+
+// the options above, with the usage message headings removed
+static struct option real_long_options[sizeof(long_options) / sizeof(long_options[0])];
+
 void usage(char **argv) {
-	fprintf(stderr, "Usage: %s -o newtile.pbf.gz tile.pbf.gz oz/ox/oy nz/nx/ny\n", argv[0]);
+	static const char *const forms[] = {
-	fprintf(stderr, "to create tile nz/nx/ny from tile oz/ox/oy\n");
+		"[options] tile.pbf.gz oz/ox/oy nz/nx/ny",
-	fprintf(stderr, "Usage: %s -o newtile.pbf.gz -t nz/nx/ny tile.pbf.gz oz/ox/oy tile2.pbf.gz oz2/ox2/oy2\n", argv[0]);
+		"[options] --source-tile=nz/nx/ny tile.pbf.gz oz/ox/oy ...",
-	fprintf(stderr, "to create tile nz/nx/ny from tiles oz/ox/oy and oz2/ox2/oy2\n");
+		NULL,
+	};
+	static const struct usage_required_option required[] = {
+		{"output", "newtile.pbf.gz", 0},
+		{NULL, NULL, 0},
+	};
+
+	print_usage(stderr, argv[0], forms, long_options, required);
+	fprintf(stderr, "\nThe tile nz/nx/ny is created from the tile or tiles oz/ox/oy that contain it.\n");
+	fprintf(stderr, "In the second form, each source tile is named by a file name and a z/x/y pair.\n");
 	exit(EXIT_FAILURE);
 }
 
@@ -67,41 +117,11 @@ int main(int argc, char **argv) {
 
 	std::vector<input_tile> sources;
 
-	struct option long_options[] = {
+	strip_usage_headings(long_options, real_long_options);
-		{"include", required_argument, 0, 'y'},
+	std::string getopt_str = getopt_string(real_long_options);
-		{"exclude", required_argument, 0, 'x'},
-		{"exclude-prefix", required_argument, 0, 'x' & 0x1F},
-		{"full-detail", required_argument, 0, 'd'},
-		{"buffer", required_argument, 0, 'b'},
-		{"output", required_argument, 0, 'o'},
-		{"filter-points-multiplier", no_argument, 0, 'm'},
-		{"feature-filter", required_argument, 0, 'j'},
-		{"feature-filter-file", required_argument, 0, 'J'},
-		{"preserve-input-order", no_argument, 0, 'o' & 0x1F},
-		{"accumulate-attribute", required_argument, 0, 'E'},
-		{"unidecode-data", required_argument, 0, 'u' & 0x1F},
-		{"line-simplification", required_argument, 0, 'S'},
-		{"tiny-polygon-size", required_argument, 0, 's' & 0x1F},
-		{"source-tile", required_argument, 0, 't'},
-		{"no-tile-compression", no_argument, 0, 'd' & 0x1F},
-		{"deduplicate-by-id", no_argument, 0, 'i' & 0x1F},
-
-		{0, 0, 0, 0},
-	};
-
-	std::string getopt_str;
-	for (size_t lo = 0; long_options[lo].name != NULL; lo++) {
-		if (long_options[lo].val > ' ') {
-			getopt_str.push_back(long_options[lo].val);
-
-			if (long_options[lo].has_arg == required_argument) {
-				getopt_str.push_back(':');
-			}
-		}
-	}
 
 	int option_index = 0;
-	while ((i = getopt_long(argc, argv, getopt_str.c_str(), long_options, &option_index)) != -1) {
+	while ((i = getopt_long(argc, argv, getopt_str.c_str(), real_long_options, &option_index)) != -1) {
 		switch (i) {
 		case 'y':
 			keep.insert(optarg);
@@ -181,6 +201,11 @@ int main(int argc, char **argv) {
 	std::vector<input_tile> its;
 	int nz, nx, ny;
 
+	if (outfile == NULL) {
+		fprintf(stderr, "%s: must specify -o newtile.pbf.gz\n", argv[0]);
+		usage(argv);
+	}
+
 	if (outtile == NULL) {	// single input
 		if (argc - optind != 3) {
 			fprintf(stderr, "Wrong number of arguments\n");

serial.cpp

@@ -665,10 +665,18 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::
 		// VT_POINT extent will be calculated in write_tile from the distance between adjacent features.
 	}
 
-	if (extent <= LLONG_MAX) {
+	// Clamp before converting, since converting a double that is out of range
+	// for a long long is undefined. The bounds are asymmetric: LLONG_MAX is not
+	// representable as a double and rounds up to 2^63, so the upper bound has to
+	// be exclusive, while LLONG_MIN is exactly -2^63 and so can be included.
+	// Areas are signed, so holes that outweigh their rings can make this
+	// negative.
+	if (extent >= (double) LLONG_MIN && extent < (double) LLONG_MAX) {
 		sf.extent = (long long) extent;
+	} else if (extent < 0) {
+		sf.extent = LLONG_MIN;
 	} else {
-		sf.extent = LLONG_MAX;
+		sf.extent = LLONG_MAX;  // also the NaN case
 	}
 
 	if (sst->want_dist && sf.t == VT_POLYGON) {

tests/flatgeobuf/numeric-properties.json

@@ -0,0 +1,5 @@
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "numeric-properties", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "byte_min": -128, "ubyte_max": 255, "bool_value": true, "short_min": -32768, "ushort_max": 65535, "int_min": -2147483648, "uint_max": 4294967295, "long_min": -9223372036854775808, "ulong_max": 18446744073709551615, "float_value": 1.25, "double_value": 12345.678901234, "guid_value": "123e4567-e89b-12d3-a456-426614174000" }, "geometry": { "type": "Point", "coordinates": [ 0.000000, 0.000000 ] } }
+] }
+] }

tests/minzoom-variable-depth/in.json

@@ -0,0 +1,2 @@
+{"type":"Feature","id":1,"tippecanoe":{"minzoom":10},"properties":{"depth":10},"geometry":{"type":"Point","coordinates":[0.05,0.05]}}
+{"type":"Feature","id":2,"tippecanoe":{"minzoom":11},"properties":{"depth":11},"geometry":{"type":"Point","coordinates":[0.06,0.06]}}

tests/minzoom-variable-depth/out/-Z10_-z11_--generate-variable-depth-tile-pyramid.json

@@ -0,0 +1,28 @@
+{ "type": "FeatureCollection", "properties": {
+"antimeridian_adjusted_bounds": "0.050000,0.050000,0.060000,0.060000",
+"bounds": "0.050000,0.050000,0.060000,0.060000",
+"center": "0.060000,0.060000,11",
+"description": "tests/minzoom-variable-depth/out/-Z10_-z11_--generate-variable-depth-tile-pyramid.json.check.mbtiles",
+"format": "pbf",
+"generator_options": "./tippecanoe -q -a@ -f -o tests/minzoom-variable-depth/out/-Z10_-z11_--generate-variable-depth-tile-pyramid.json.check.mbtiles -Z10 -z11 --generate-variable-depth-tile-pyramid tests/minzoom-variable-depth/in.json",
+"json": "{\"vector_layers\":[{\"id\":\"in\",\"description\":\"\",\"minzoom\":10,\"maxzoom\":11,\"fields\":{\"depth\":\"Number\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"in\",\"count\":2,\"geometry\":\"Point\",\"attributeCount\":1,\"attributes\":[{\"attribute\":\"depth\",\"count\":2,\"type\":\"number\",\"values\":[10,11],\"min\":10,\"max\":11}]}]}}",
+"maxzoom": "11",
+"minzoom": "10",
+"name": "tests/minzoom-variable-depth/out/-Z10_-z11_--generate-variable-depth-tile-pyramid.json.check.mbtiles",
+"type": "overlay",
+"version": "2"
+}, "features": [
+{ "type": "FeatureCollection", "properties": { "zoom": 10, "x": 512, "y": 511 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "id": 1, "properties": { "depth": 10 }, "geometry": { "type": "Point", "coordinates": [ 0.050039, 0.049953 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 11, "x": 1024, "y": 1023 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "id": 1, "properties": { "depth": 10 }, "geometry": { "type": "Point", "coordinates": [ 0.049996, 0.049996 ] } }
+,
+{ "type": "Feature", "id": 2, "properties": { "depth": 11 }, "geometry": { "type": "Point", "coordinates": [ 0.059996, 0.059996 ] } }
+] }
+] }
+] }

tile-join.cpp

@@ -44,6 +44,7 @@
 #include "geometry.hpp"
 #include "thread.hpp"
 #include "platform.hpp"
+#include "usage.hpp"
 
 int pk = false;
 int pC = false;
@@ -892,7 +893,7 @@ void *join_worker(void *v) {
 }
 
 void dispatch_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, const char *outdir, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, sqlite3 *db, std::set<std::string> &exclude, std::set<std::string> &include, int ifmatched, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, json_object *filter, struct tileset_reader *readers, double *minlat, double *minlon, double *maxlat, double *maxlon, double *minlon2, double *maxlon2) {
-	pthread_t pthreads[CPUS];
+	std::vector<pthread_t> pthreads(CPUS);
 	std::vector<arg> args;
 
 	for (size_t i = 0; i < CPUS; i++) {
@@ -1253,8 +1254,81 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
 	}
 }
 
+static const struct option long_options[] = {
+	{"Output tileset", 0, 0, 0},
+	{"output", required_argument, 0, 'o'},
+	{"output-to-directory", required_argument, 0, 'e'},
+	{"force", no_argument, 0, 'f'},
+
+	{"Tileset description and attribution", 0, 0, 0},
+	{"name", required_argument, 0, 'n'},
+	{"attribution", required_argument, 0, 'A'},
+	{"description", required_argument, 0, 'N'},
+
+	{"Input tilesets", 0, 0, 0},
+	{"read-from", required_argument, 0, 'r'},
+
+	{"Zoom levels", 0, 0, 0},
+	{"maximum-zoom", required_argument, 0, 'z'},
+	{"minimum-zoom", required_argument, 0, 'Z'},
+	{"overzoom", no_argument, 0, 'O'},
+	{"buffer", required_argument, 0, 'b'},
+
+	{"Layer names", 0, 0, 0},
+	{"layer", required_argument, 0, 'l'},
+	{"exclude-layer", required_argument, 0, 'L'},
+	{"rename-layer", required_argument, 0, 'R'},
+
+	{"Joining with a CSV file", 0, 0, 0},
+	{"csv", required_argument, 0, 'c'},
+	{"if-matched", no_argument, 0, 'i'},
+	{"empty-csv-columns-are-null", no_argument, &pe, 1},
+
+	{"Filtering feature attributes", 0, 0, 0},
+	{"exclude", required_argument, 0, 'x'},
+	{"include", required_argument, 0, 'y'},
+	{"exclude-all", no_argument, 0, 'X'},
+	{"exclude-all-tile-attributes", no_argument, 0, '~'},
+	{"exclude-all-tile-geometries", no_argument, 0, '~'},
+
+	{"Filtering features by attributes", 0, 0, 0},
+	{"feature-filter-file", required_argument, 0, 'J'},
+	{"feature-filter", required_argument, 0, 'j'},
+
+	{"Setting or disabling tile size limits", 0, 0, 0},
+	{"no-tile-size-limit", no_argument, &pk, 1},
+	{"no-tile-compression", no_argument, &pC, 1},
+	{"no-tile-stats", no_argument, &pg, 1},
+	{"tile-stats-attributes-limit", required_argument, 0, '~'},
+	{"tile-stats-sample-values-limit", required_argument, 0, '~'},
+	{"tile-stats-values-limit", required_argument, 0, '~'},
+
+	{"Progress indicator", 0, 0, 0},
+	{"quiet", no_argument, 0, 'q'},
+
+	{"", 0, 0, 0},
+	{"prevent", required_argument, 0, 'p'},
+	{"unidecode-data", required_argument, 0, '~'},
+
+	{0, 0, 0, 0},
+};
+
+// the options above, with the usage message headings removed
+static struct option real_long_options[sizeof(long_options) / sizeof(long_options[0])];
+
 void usage(char **argv) {
-	fprintf(stderr, "Usage: %s [-f] [-i] [-pk] [-pC] [-c joins.csv] [-X] [-x exclude ...] [-y include ...] [-r inputfile.txt ] -o new.mbtiles source.mbtiles ...\n", argv[0]);
+	static const char *const forms[] = {
+		"[options] source.mbtiles ...",
+		"[options] --read-from=inputfile.txt",
+		NULL,
+	};
+	static const struct usage_required_option required[] = {
+		{"output", "new.mbtiles", 1},
+		{"output-to-directory", "directory", 1},
+		{NULL, NULL, 0},
+	};
+
+	print_usage(stderr, argv[0], forms, long_options, required);
 	exit(EXIT_ARGS);
 }
 
@@ -1298,57 +1372,8 @@ int main(int argc, char **argv) {
 
 	std::string set_name, set_description, set_attribution;
 
-	struct option long_options[] = {
+	strip_usage_headings(long_options, real_long_options);
-		{"output", required_argument, 0, 'o'},
+	std::string getopt_str = getopt_string(real_long_options);
-		{"output-to-directory", required_argument, 0, 'e'},
-		{"force", no_argument, 0, 'f'},
-		{"overzoom", no_argument, 0, 'O'},
-		{"buffer", required_argument, 0, 'b'},
-		{"if-matched", no_argument, 0, 'i'},
-		{"attribution", required_argument, 0, 'A'},
-		{"name", required_argument, 0, 'n'},
-		{"description", required_argument, 0, 'N'},
-		{"prevent", required_argument, 0, 'p'},
-		{"csv", required_argument, 0, 'c'},
-		{"exclude", required_argument, 0, 'x'},
-		{"exclude-all", no_argument, 0, 'X'},
-		{"include", required_argument, 0, 'y'},
-		{"exclude-all-tile-attributes", no_argument, 0, '~'},
-		{"exclude-all-tile-geometries", no_argument, 0, '~'},
-		{"layer", required_argument, 0, 'l'},
-		{"exclude-layer", required_argument, 0, 'L'},
-		{"quiet", no_argument, 0, 'q'},
-		{"maximum-zoom", required_argument, 0, 'z'},
-		{"minimum-zoom", required_argument, 0, 'Z'},
-		{"feature-filter-file", required_argument, 0, 'J'},
-		{"feature-filter", required_argument, 0, 'j'},
-		{"rename-layer", required_argument, 0, 'R'},
-		{"read-from", required_argument, 0, 'r'},
-
-		{"use-attribute-for-id", required_argument, 0, '~'},
-
-		{"no-tile-size-limit", no_argument, &pk, 1},
-		{"no-tile-compression", no_argument, &pC, 1},
-		{"empty-csv-columns-are-null", no_argument, &pe, 1},
-		{"no-tile-stats", no_argument, &pg, 1},
-		{"tile-stats-attributes-limit", required_argument, 0, '~'},
-		{"tile-stats-sample-values-limit", required_argument, 0, '~'},
-		{"tile-stats-values-limit", required_argument, 0, '~'},
-		{"unidecode-data", required_argument, 0, '~'},
-
-		{0, 0, 0, 0},
-	};
-
-	std::string getopt_str;
-	for (size_t lo = 0; long_options[lo].name != NULL; lo++) {
-		if (long_options[lo].val > ' ') {
-			getopt_str.push_back(long_options[lo].val);
-
-			if (long_options[lo].has_arg == required_argument) {
-				getopt_str.push_back(':');
-			}
-		}
-	}
 
 	extern int optind;
 	extern char *optarg;
@@ -1357,7 +1382,7 @@ int main(int argc, char **argv) {
 	std::string commandline = format_commandline(argc, argv);
 
 	int option_index = 0;
-	while ((i = getopt_long(argc, argv, getopt_str.c_str(), long_options, &option_index)) != -1) {
+	while ((i = getopt_long(argc, argv, getopt_str.c_str(), real_long_options, &option_index)) != -1) {
 		switch (i) {
 		case 0:
 			break;
@@ -1502,7 +1527,7 @@ int main(int argc, char **argv) {
 			break;
 
 		case '~': {
-			const char *opt = long_options[option_index].name;
+			const char *opt = real_long_options[option_index].name;
 			if (strcmp(opt, "tile-stats-attributes-limit") == 0) {
 				max_tilestats_attributes = atoi(optarg);
 			} else if (strcmp(opt, "tile-stats-sample-values-limit") == 0) {

tile.cpp

@@ -61,9 +61,6 @@ extern "C" {
 #define COORD_OFFSET (4LL << 32)
 #define SHIFT_RIGHT(a) ((long long) std::round((double) (a) / (1LL << geometry_scale)))
 
-#define XSTRINGIFY(s) STRINGIFY(s)
-#define STRINGIFY(s) #s
-
 pthread_mutex_t db_lock = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_t var_lock = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_t task_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -541,7 +538,6 @@ struct simplification_worker_arg {
 	std::vector<std::shared_ptr<serial_feature>> *features = NULL;
 	int task = 0;
 	int tasks = 0;
-	bool trying_to_stop_early = false;
 
 	drawvec *shared_nodes;
 	node *shared_nodes_map;
@@ -676,9 +672,7 @@ static void *simplification_worker(void *v) {
 
 	for (size_t i = a->task; i < (*features).size(); i += a->tasks) {
 		double area = 0;
-		if (!a->trying_to_stop_early) {
+		area = simplify_feature(&*((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos, *(a->shared_nodes_bloom));
-			area = simplify_feature(&*((*features)[i]), *(a->shared_nodes), a->shared_nodes_map, a->nodepos, *(a->shared_nodes_bloom));
-		}
 
 		signed char t = (*features)[i]->t;
 		int z = (*features)[i]->z;
@@ -693,20 +687,18 @@ static void *simplification_worker(void *v) {
 			{
 				drawvec before = geom;
 
-				if (!a->trying_to_stop_early) {
+				// we can try scaling up because this is now tile scale
-					// we can try scaling up because this is now tile scale
+				coalesce_polygon(geom, true);
-					coalesce_polygon(geom, true);
+				if (additional[A_DEBUG_POLYGON]) {
-					if (additional[A_DEBUG_POLYGON]) {
+					check_polygon(geom);
-						check_polygon(geom);
+				}
-					}
 
-					if (geom.size() < 3) {
+				if (geom.size() < 3) {
-						if (area > 0) {
+					if (area > 0) {
-							// area is in world coordinates, calculated before scaling down
+						// area is in world coordinates, calculated before scaling down
-							geom = revive_polygon(before, area, z, out_detail);
+						geom = revive_polygon(before, area, z, out_detail);
-						} else {
+					} else {
-							geom.clear();
+						geom.clear();
-						}
 					}
 				}
 			}
@@ -929,8 +921,12 @@ struct write_tile_args {
 	long long minextent_out = 0;
 	unsigned long long mindrop_sequence = 0;
 	unsigned long long mindrop_sequence_out = 0;
-	double minattribute = 0;
+	// the infinity that excludes nothing, for the ascending order that
-	double minattribute_out = 0;
+	// drop_by_attribute_descending also defaults to. dropping_features()
+	// reads these, so they must not start out at a value that looks like
+	// a threshold that has already been chosen.
+	double minattribute = -HUGE_VAL;
+	double minattribute_out = -HUGE_VAL;
 	std::string const *drop_by_attribute_as_needed_attribute = NULL;
 	bool drop_by_attribute_descending = false;
 	size_t tile_size_out = 0;
@@ -952,8 +948,26 @@ struct write_tile_args {
 	std::string const *shared_nodes_bloom;
 	std::set<zxy> const *skip_children;  // what is being skipped at this zoom
 	std::set<zxy> skip_children_out;     // what will be skipped in the next zoom
+	bool first_dropping_pass = false;    // is this the first pass of this zoom that is dropping features?
 };
 
+// Is this zoom level discarding features to make its tiles fit? The thresholds
+// start out excluding nothing, and only move inward once some tile in the zoom
+// fails to fit and the whole zoom is retried with a threshold for discarding
+// features. For mingap, minextent, and mindrop_sequence that starting point is
+// zero; for minattribute it is the infinity on whichever side is being kept.
+//
+// Pyramid truncation is disabled while this is true, so that every tile in the
+// zoom is reduced consistently, and tiles that were skipped because an ancestor
+// truncated its pyramid are revived.
+static bool dropping_features(unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, double minattribute, bool drop_by_attribute_descending) {
+	bool dropping_by_attribute = drop_by_attribute_descending
+					     ? minattribute != HUGE_VAL
+					     : minattribute != -HUGE_VAL;
+
+	return mingap != 0 || minextent != 0 || mindrop_sequence != 0 || dropping_by_attribute;
+}
+
 // Clips a feature's geometry to the tile bounds at the specified zoom level
 // with the specified buffer. Returns true if the feature was entirely clipped away
 // by bounding box alone; otherwise returns false.
@@ -1097,12 +1111,15 @@ static bool skip_next_feature(decompressor *geoms, std::atomic<long long> *geomp
 struct next_feature_state {
 	unsigned long long previndex = 0;
 	unsigned long long prev_not_dropped_index = 0;
+	// set when a feature excluded by tippecanoe_minzoom first appears beyond the next
+	// zoom, so variable-depth pyramids don't leaf above the feature's minzoom and prune it away
+	bool minzoom_feature_pending = false;
 };
 
 // This function is called repeatedly from write_tile() to retrieve the next feature
 // from the input stream. If the stream is at an end, it returns a feature with the
 // geometry type set to -2.
-static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, std::atomic<bool> *within, compressor **geomfile, std::atomic<long long> *geompos, long long start_geompos[], std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool compressed, multiplier_state *multiplier_state, std::shared_ptr<std::string> &tile_stringpool, std::vector<std::string> const &unidecode_data, next_feature_state &next_feature_state, double droprate) {
+static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, std::atomic<bool> *within, compressor **geomfile, std::atomic<long long> *geompos, long long start_geompos[], std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, json_object *filter, const char *global_stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time, bool write_children, bool compressed, multiplier_state *multiplier_state, std::shared_ptr<std::string> &tile_stringpool, std::vector<std::string> const &unidecode_data, next_feature_state &next_feature_state, double droprate) {
 	double extra_multiplier_zooms = log(retain_points_multiplier) / log(droprate);
 
 	while (1) {
@@ -1191,7 +1208,7 @@ static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *
 			// XXX should continue, but affects test outputs
 		}
 
-		if (first_time && pass == 0) { /* only write out the next zoom once, even if we retry */
+		if (first_time && write_children) { /* only write out the next zoom once, even if we retry */
 			if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) {
 				rewrite(sf, z, nextzoom, maxzoom, tx, ty, buffer, within, geompos, start_geompos, geomfile, fname, child_shards, max_zoom_increment, sf.segment, initial_x, initial_y);
 			}
@@ -1202,6 +1219,9 @@ static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *
 		}
 
 		if (sf.tippecanoe_minzoom != -1 && z < sf.tippecanoe_minzoom) {
+			// a leaf at z carries only z-visible content, so an excluded feature at
+			// any deeper minzoom (even z + 1) must block leafing here
+			next_feature_state.minzoom_feature_pending = true;
 			continue;
 		}
 		if (sf.tippecanoe_maxzoom != -1 && z > sf.tippecanoe_maxzoom) {
@@ -1353,6 +1373,7 @@ struct run_prefilter_args {
 	json_object *filter = NULL;
 	std::vector<std::string> const *unidecode_data;
 	bool first_time = false;
+	bool write_children = false;
 	bool compressed = false;
 	double droprate = 1;
 };
@@ -1365,7 +1386,7 @@ void *run_prefilter(void *v) {
 	next_feature_state next_feature_state;
 
 	while (1) {
-		serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->start_geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->global_stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->compressed, &multiplier_state, tile_stringpool, *(rpa->unidecode_data), next_feature_state, rpa->droprate);
+		serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->start_geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->global_stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time, rpa->write_children, rpa->compressed, &multiplier_state, tile_stringpool, *(rpa->unidecode_data), next_feature_state, rpa->droprate);
 		if (sf.t < 0) {
 			break;
 		}
@@ -1641,7 +1662,7 @@ void skip_tile(decompressor *geoms, std::atomic<long long> *geompos_in, bool com
 	}
 }
 
-long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, double minattribute, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector<std::string> const &unidecode_data, long long estimated_complexity, std::set<zxy> &skip_children_out) {
+long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, double minattribute, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector<std::string> const &unidecode_data, long long estimated_complexity, bool write_children, std::set<zxy> &skip_children_out) {
 	double merge_fraction = 1;
 	double mingap_fraction = 1;
 	double minextent_fraction = 1;
@@ -1677,7 +1698,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 	int first_detail = detail, second_detail = detail - 1;
 	bool trying_to_stop_early = false;
 	bool can_stop_early = true;
-	if (additional[A_VARIABLE_DEPTH_PYRAMID]) {
+	if (additional[A_VARIABLE_DEPTH_PYRAMID] && !dropping_features(mingap, minextent, mindrop_sequence, minattribute, arg->drop_by_attribute_descending)) {
 		// If we are trying to stop early, there is an extra first pass with full+extra detail,
 		// and which loops if everything doesn't fit rather than trying to drop or union features.
 
@@ -1741,8 +1762,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 
 		key_pool key_pool;
 
-		std::atomic<bool> within[child_shards];
+		std::vector<std::atomic<bool> > within(child_shards);
-		long long start_geompos[child_shards];
+		std::vector<long long> start_geompos(child_shards);
 		for (size_t i = 0; i < (size_t) child_shards; i++) {
 			within[i] = false;
 			start_geompos[i] = -1;
@@ -1807,10 +1828,10 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			rpa.along = along;
 			rpa.alongminus = alongminus;
 			rpa.buffer = buffer;
-			rpa.within = within;
+			rpa.within = within.data();
 			rpa.geomfile = geomfile;
 			rpa.geompos = geompos;
-			rpa.start_geompos = start_geompos;
+			rpa.start_geompos = start_geompos.data();
 			rpa.oprogress = &oprogress;
 			rpa.todo = todo;
 			rpa.fname = fname;
@@ -1822,6 +1843,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			rpa.filter = filter;
 			rpa.unidecode_data = &unidecode_data;
 			rpa.first_time = first_time;
+			rpa.write_children = write_children;
 			rpa.compressed = compressed_input;
 			rpa.droprate = arg->droprate;
 
@@ -1855,7 +1877,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			ssize_t which_serial_feature = -1;
 
 			if (prefilter == NULL) {
-				sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, start_geompos, &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, compressed_input, &multiplier_state, tile_stringpool, unidecode_data, next_feature_state, arg->droprate);
+				sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within.data(), geomfile, geompos, start_geompos.data(), &oprogress, todo, fname, child_shards, filter, global_stringpool, pool_off, layer_unmaps, first_time, write_children, compressed_input, &multiplier_state, tile_stringpool, unidecode_data, next_feature_state, arg->droprate);
 			} else {
 				sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL, key_pool);
 			}
@@ -2245,6 +2267,12 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					sf.clustered = 0;
 					sf.tile_stringpool = tile_stringpool;
 
+					if (trying_to_stop_early && line_detail == first_detail) {
+						// only remove collinearities;
+						// leave all other vertices for extreme overzooming
+						sf.simplification = 0;
+					}
+
 					if (line_detail == detail && extra_detail >= 0 && z == maxzoom) {
 						sf.extra_detail = extra_detail;
 						// maximum allowed coordinate delta in geometries is 2^31 - 1
@@ -2315,7 +2343,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			if (within[j]) {
 				long long estimated_complexity_out = geompos[j] - start_geompos[j];
 
-				if (dropping_by_rate) {
+				if (dropping_by_rate || next_feature_state.minzoom_feature_pending) {
 					// large enough to make it not try to stop early
 					estimated_complexity_out = 1LL << 32;
 				}
@@ -2392,7 +2420,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				if (p.clustered > 0) {
 					serial_val sv, sv2, sv3, sv4;
 					long long point_count = p.clustered + 1;
-					char abbrev[20];  // to_string(LLONG_MAX).length() / 1000 + 1;
+					char abbrev[24];  // fits "%lld" of any long long, including the sign and the NUL
 
 					p.full_keys.push_back(key_pool.pool("clustered"));
 					sv.type = mvt_bool;
@@ -2442,7 +2470,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			}
 
 			{
-				pthread_t pthreads[tasks];
+				std::vector<pthread_t> pthreads(tasks);
 				std::vector<simplification_worker_arg> args;
 				args.resize(tasks);
 				for (int i = 0; i < tasks; i++) {
@@ -2453,7 +2481,6 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					args[i].shared_nodes_map = shared_nodes_map;
 					args[i].nodepos = nodepos;
 					args[i].shared_nodes_bloom = &shared_nodes_bloom;
-					args[i].trying_to_stop_early = trying_to_stop_early;
 
 					if (tasks > 1) {
 						if (thread_create(&pthreads[i], NULL, simplification_worker, &args[i]) != 0) {
@@ -2678,7 +2705,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			oprogress = progress;
 		}
 
-		if (trying_to_stop_early && line_detail == first_detail && !can_stop_early) {
+		if (trying_to_stop_early && line_detail == first_detail && (!can_stop_early || next_feature_state.minzoom_feature_pending)) {
 			// didn't work, try a lower detail
 			continue;
 		}
@@ -2880,7 +2907,6 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 
 				if (trying_to_stop_early && line_detail == first_detail) {
 					// didn't work, try a lower detail
-					detail_reduced++;
 					continue;
 				}
 
@@ -3030,9 +3056,10 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					exit(EXIT_PTHREAD);
 				}
 
-				if (trying_to_stop_early && line_detail == first_detail) {
+				if (trying_to_stop_early && line_detail == first_detail &&
-					// We succeeded in stopping early.
+				    !next_feature_state.minzoom_feature_pending) {
-					// Prune the child tiles.
+					// We succeeded in stopping early (and no excluded feature is
+					// waiting for a deeper zoom). Prune the child tiles.
 
 					strategy.truncated_zooms++;
 					skip_children_out.insert(zxy(z, tx, ty));
@@ -3142,12 +3169,41 @@ exit(EXIT_IMPOSSIBLE);
 			long long len;
 
 			struct zxy parent(z - 1, x / 2, y / 2);
+			bool skip = false;
+
+			// Normally a tile writes the geometry for its children on pass 0, and the
+			// later passes of the same zoom, which are only retries with new thresholds,
+			// must not write it again.
+			bool write_children = (arg->pass == 0);
+
 			if (arg->skip_children->count(parent) > 0) {
+				if (dropping_features(arg->mingap, arg->minextent, arg->mindrop_sequence, arg->minattribute, arg->drop_by_attribute_descending)) {
+					// The parent truncated its pyramid here, but the zoom has since had to
+					// start dropping features, so the truncation no longer holds and this
+					// tile has to be written after all. Its geometry is still in the stream.
+					skip = false;
+
+					// It was skipped on every earlier pass of this zoom, so its children
+					// have never been written out. Write them on the first pass that
+					// actually tiles it, rather than on pass 0 as usual. The thresholds
+					// only ever increase within a zoom, so that is exactly the pass on
+					// which this zoom started dropping.
+					write_children = arg->first_dropping_pass;
+				} else {
+					// skip if the parent tile finished at the last zoom level and we aren't trying to drop
+					skip = true;
+				}
+			} else {
+				// do the tile if the parent didn't finish early
+				skip = false;
+			}
+
+			if (skip) {
 				skip_tile(&dc, &geompos, arg->compressed);
 				len = 1;
 			} else {
 				arg->wrote_zoom = z;
-				len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->minattribute, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out);
+				len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->minattribute, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, write_children, arg->skip_children_out);
 			}
 
 			if (pthread_mutex_lock(&var_lock) != 0) {
@@ -3245,28 +3301,27 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 	for (z = iz; z <= maxzoom; z++) {
 		std::atomic<long long> most(0);
 
-		compressor compressors[TEMP_FILES];
+		std::vector<compressor> compressors(TEMP_FILES);
-		compressor *sub[TEMP_FILES];
+		std::vector<compressor *> sub(TEMP_FILES);
-		std::atomic<long long> subpos[TEMP_FILES];
+		std::vector<std::atomic<long long> > subpos(TEMP_FILES);
-		int subfd[TEMP_FILES];
+		std::vector<int> subfd(TEMP_FILES);
 		for (size_t j = 0; j < TEMP_FILES; j++) {
-			char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX" XSTRINGIFY(INT_MAX)) + 1];
+			std::string geomname = std::string(tmpdir) + "/geom" + std::to_string(j) + ".XXXXXXXX";
-			snprintf(geomname, sizeof(geomname), "%s/geom%zu.XXXXXXXX", tmpdir, j);
+			subfd[j] = mkstemp_cloexec(&geomname[0]);
-			subfd[j] = mkstemp_cloexec(geomname);
+			// printf("%s\n", geomname.c_str());
-			// printf("%s\n", geomname);
 			if (subfd[j] < 0) {
-				perror(geomname);
+				perror(geomname.c_str());
 				exit(EXIT_OPEN);
 			}
-			FILE *fp = fopen_oflag(geomname, "wb", O_WRONLY | O_CLOEXEC);
+			FILE *fp = fopen_oflag(geomname.c_str(), "wb", O_WRONLY | O_CLOEXEC);
 			if (fp == NULL) {
-				perror(geomname);
+				perror(geomname.c_str());
 				exit(EXIT_OPEN);
 			}
 			compressors[j] = compressor(fp);
 			sub[j] = &compressors[j];
 			subpos[j] = 0;
-			unlink(geomname);
+			unlink(geomname.c_str());
 		}
 
 		size_t useful_threads = 0;
@@ -3332,9 +3387,17 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 		size_t zoom_tile_size = 0;
 		size_t zoom_feature_count = 0;
 		std::set<zxy> skip_children_out;
+		bool was_dropping = false;
 
 		for (size_t pass = 0;; pass++) {
-			pthread_t pthreads[threads];
+			// Tiles that were skipped because an ancestor truncated its pyramid are
+			// revived on the pass where this zoom starts dropping features, and that
+			// is the pass on which they have to write out their children.
+			bool is_dropping = dropping_features(zoom_mingap, zoom_minextent, zoom_mindrop_sequence, zoom_minattribute, drop_by_attribute_descending);
+			bool first_dropping_pass = is_dropping && !was_dropping;
+			was_dropping = is_dropping;
+
+			std::vector<pthread_t> pthreads(threads);
 			std::vector<write_tile_args> args;
 			args.resize(threads);
 			std::atomic<int> running(threads);
@@ -3353,8 +3416,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 				args[thread].outdir = outdir;
 				args[thread].buffer = buffer;
 				args[thread].fname = fname;
-				args[thread].geomfile = sub + thread * (TEMP_FILES / threads);
+				args[thread].geomfile = sub.data() + thread * (TEMP_FILES / threads);
-				args[thread].geompos = subpos + thread * (TEMP_FILES / threads);
+				args[thread].geompos = subpos.data() + thread * (TEMP_FILES / threads);
 				args[thread].todo = todo;
 				args[thread].along = &along;  // locked with var_lock
 				args[thread].gamma = zoom_gamma;
@@ -3415,6 +3478,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
 				args[thread].shared_nodes_bloom = &shared_nodes_bloom;
 				args[thread].skip_children = &skip_children;
 				args[thread].skip_children_out.clear();
+				args[thread].first_dropping_pass = first_dropping_pass;
 
 				if (thread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) {
 					perror("pthread_create");

unit.cpp

@@ -136,3 +136,36 @@ TEST_CASE("line_is_too_small") {
 	dv.emplace_back(VT_LINETO, -51864809, 2683873977);
 	REQUIRE(line_is_too_small(dv, 0, 10));
 }
+
+TEST_CASE("Polygon cleaning drops a hole that no ring can parent", "[wagyu]") {
+	// Two mutually reversed self-intersecting rings whose union leaves a hole
+	// that wagyu's topology correction cannot assign to any surviving parent
+	// ring (found by fuzzing; same failure as mapbox/tippecanoe#761). Without
+	// the fix in mapbox/geometry/wagyu/topology_correction.hpp, this exits
+	// through the "Could not properly place hole to a parent." handler in
+	// clean_or_clip_poly instead of returning.
+	static const std::vector<std::vector<std::pair<long long, long long>>> rings = {
+	    {{0, 5}, {5, 4}, {5, 1}, {4, 4}, {4, 2}, {7, 1}, {0, 5}},
+	    {{0, 5}, {7, 1}, {4, 2}, {4, 4}, {5, 1}, {5, 4}, {0, 0}, {0, 5}},
+	};
+
+	drawvec geom;
+	for (auto const &ring : rings) {
+		for (size_t i = 0; i < ring.size(); i++) {
+			geom.push_back(draw(i == 0 ? VT_MOVETO : VT_LINETO, ring[i].first, ring[i].second));
+		}
+	}
+
+	drawvec out = clean_or_clip_poly(geom, 0, 0, false, false);
+
+	// The regression signal is getting here at all: without the fix,
+	// clean_or_clip_poly exits the process from its wagyu error handler.
+	SUCCEED("clean_or_clip_poly returned");
+
+	// Anything that survives must be sanely wound: first ring positive.
+	if (out.size() > 0) {
+		size_t j = 1;
+		while (j < out.size() && out[j].op == VT_LINETO) j++;
+		REQUIRE(get_area(out, 0, j) > 0);
+	}
+}

usage.cpp

@@ -0,0 +1,133 @@
+#include <string.h>
+#include <set>
+#include <string>
+#include "usage.hpp"
+
+// Options are wrapped to fit within this many columns
+#define USAGE_WIDTH 80
+
+// The indentation of the continuation lines of the option list
+#define USAGE_INDENT 8
+
+std::string getopt_string(const struct option *long_options) {
+	std::string getopt_str;
+
+	for (size_t lo = 0; long_options[lo].name != NULL; lo++) {
+		if (long_options[lo].val > ' ') {
+			getopt_str.push_back(long_options[lo].val);
+
+			if (long_options[lo].has_arg == required_argument) {
+				getopt_str.push_back(':');
+			}
+		}
+	}
+
+	return getopt_str;
+}
+
+void strip_usage_headings(const struct option *long_options, struct option *real_long_options) {
+	size_t out = 0;
+
+	for (size_t lo = 0; long_options[lo].name != NULL; lo++) {
+		if (long_options[lo].val != 0) {
+			real_long_options[out++] = long_options[lo];
+		}
+	}
+
+	real_long_options[out] = {0, 0, 0, 0};
+}
+
+// The entry for `name` in the list of options that must be specified,
+// or NULL if it is an optional option
+static const struct usage_required_option *required_for(const char *name, const struct usage_required_option *required) {
+	for (size_t i = 0; required != NULL && required[i].name != NULL; i++) {
+		if (strcmp(required[i].name, name) == 0) {
+			return &required[i];
+		}
+	}
+
+	return NULL;
+}
+
+// "--option", or "--option=placeholder" if the option takes an argument
+static std::string option_text(const struct option *opt, const struct usage_required_option *req) {
+	std::string text = std::string("--") + opt->name;
+
+	if (opt->has_arg != no_argument) {
+		text += "=";
+		text += (req != NULL && req->placeholder != NULL) ? req->placeholder : "...";
+	}
+
+	return text;
+}
+
+// The alternatives that `req` belongs to, as "(--this=... | --that=...)"
+static std::string alternation_text(const struct option *long_options, const struct usage_required_option *required, int alternation) {
+	std::string text;
+	size_t found = 0;
+
+	for (size_t lo = 0; long_options[lo].name != NULL && long_options[lo].name[0] != '\0'; lo++) {
+		const struct usage_required_option *req = required_for(long_options[lo].name, required);
+
+		if (req != NULL && req->alternation == alternation) {
+			if (found++ > 0) {
+				text += " | ";
+			}
+
+			text += option_text(&long_options[lo], req);
+		}
+	}
+
+	if (found > 1) {
+		text = "(" + text + ")";
+	}
+
+	return text;
+}
+
+void print_usage(FILE *out, const char *program, const char *const *forms,
+		 const struct option *long_options,
+		 const struct usage_required_option *required) {
+	for (size_t f = 0; forms[f] != NULL; f++) {
+		const char *lead = (f == 0) ? "Usage: " : "\n   or: ";
+		fprintf(out, "%s%s %s", lead, program, forms[f]);
+	}
+
+	// whatever the forms took up, the option list starts on a line of its own
+	size_t width = USAGE_WIDTH;
+	std::set<int> alternations_listed;
+
+	for (size_t lo = 0; long_options[lo].name != NULL && long_options[lo].name[0] != '\0'; lo++) {
+		if (long_options[lo].val == 0) {
+			fprintf(out, "\n  %s\n%*s", long_options[lo].name, USAGE_INDENT, "");
+			width = USAGE_INDENT;
+			continue;
+		}
+
+		const struct usage_required_option *req = required_for(long_options[lo].name, required);
+		std::string text;
+
+		if (req == NULL) {
+			text = "[" + option_text(&long_options[lo], NULL) + "]";
+		} else if (req->alternation == 0) {
+			text = option_text(&long_options[lo], req);
+		} else {
+			if (alternations_listed.count(req->alternation) != 0) {
+				continue;  // already listed with the first of its alternatives
+			}
+			alternations_listed.insert(req->alternation);
+
+			text = alternation_text(long_options, required, req->alternation);
+		}
+
+		if (width + 1 + text.size() >= USAGE_WIDTH) {
+			fprintf(out, "\n%*s", USAGE_INDENT, "");
+			width = USAGE_INDENT;
+		}
+
+		fprintf(out, " %s", text.c_str());
+		width += 1 + text.size();
+	}
+
+	fprintf(out, "\n");
+}

usage.hpp

@@ -0,0 +1,52 @@
+#ifndef USAGE_HPP
+#define USAGE_HPP
+
+#include <stdio.h>
+#include <getopt.h>
+#include <string>
+
+// An option that must be specified rather than being optional, and the
+// placeholder to show for its argument in the usage message.
+//
+// Options that share the same non-zero `alternation` are alternatives to
+// each other: one of them must be specified, but not more than one, and
+// they are listed together as `(--this=... | --that=...)`.
+struct usage_required_option {
+	const char *name;
+	const char *placeholder;
+	int alternation;
+};
+
+// Returns the short option string to pass to getopt_long() for the
+// options in `long_options`, so that the two can't disagree about
+// which short options exist or take arguments.
+std::string getopt_string(const struct option *long_options);
+
+// Copies `long_options` to `real_long_options`, leaving out the headings
+// of the usage message, which are not real options and so must not be
+// passed on to getopt_long(). The destination must be at least as large
+// as the source.
+void strip_usage_headings(const struct option *long_options, struct option *real_long_options);
+
+// Prints a usage message for `program` to `out`:
+//
+//	Usage: program forms[0]
+//	   or: program forms[1]
+//	        [--some-option] [--another-option=...] ...
+//
+// where `forms` is a NULL-terminated list of the ways the non-option
+// arguments can be given, and the list of options is derived from
+// `long_options`, the same table that is passed to getopt_long(), so that
+// the message stays in sync with the options that are really accepted.
+//
+// Options named in `required` (a list terminated by a NULL name, or NULL
+// if there are none) are shown without brackets, using the placeholder
+// given there for their argument, and grouped with any alternatives to
+// them. An entry in `long_options` with no `val` is printed as a heading
+// for the options that follow it, and an entry with an empty name ends
+// the listing, hiding any options after it.
+void print_usage(FILE *out, const char *program, const char *const *forms,
+		 const struct option *long_options,
+		 const struct usage_required_option *required);
+
+#endif

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.80.0"
+#define VERSION "v2.81.0"
 
 #endif