Minimize external dependencies for Geobuf testing

.travis.yml

@@ -68,7 +68,8 @@ install:
  - BUILDTYPE=${BUILDTYPE} make -j
 
 script:
- - BUILDTYPE=${BUILDTYPE} make test
+ - npm install geobuf
+ - BUILDTYPE=${BUILDTYPE} make test geobuf-test
  - if [ -n "${COVERAGE}" ]; then
        /usr/bin/llvm-cov-3.5 -lp *.o;
        pip install --user cpp-coveralls;

Makefile

@@ -19,7 +19,7 @@ else
 	FINAL_FLAGS := -g $(WARNING_FLAGS) $(DEBUG_FLAGS)
 endif
 
-all: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit
+all: tippecanoe tippecanoe-enumerate tippecanoe-decode tile-join unit geojson2nd
 
 docs: man/tippecanoe.1
 
@@ -58,6 +58,9 @@ tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o
 tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
+geojson2nd: geojson2nd.o jsonpull/jsonpull.o
+	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
+
 unit: unit.o text.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
@@ -88,12 +91,12 @@ test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test p
 	cmp $@.out $(patsubst %.check,%,$@)
 	rm $@.out $@.mbtiles
 
-geobuf-test: $(addsuffix .checkbuf,$(TESTS))
+geobuf-test: geojson2nd $(addsuffix .checkbuf,$(TESTS))
 
 # XXX Use proper makefile rules instead of a for loop
 %.json.checkbuf:
-	for i in $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json); do ./tests/fc-wrap $$i | json2geobuf > $$i.pbf; done
+	for i in $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json); do ./geojson2nd -w $$i | json2geobuf > $$i.geobuf; done
-	./tippecanoe -aD -f -o $@.mbtiles $(subst @,:,$(subst %,/,$(subst _, ,$(patsubst %.json.checkbuf,%,$(word 4,$(subst /, ,$@)))))) $(addsuffix .pbf,$(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json)) < /dev/null
+	./tippecanoe -aD -f -o $@.mbtiles $(subst @,:,$(subst %,/,$(subst _, ,$(patsubst %.json.checkbuf,%,$(word 4,$(subst /, ,$@)))))) $(addsuffix .geobuf,$(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json)) < /dev/null
 	./tippecanoe-decode $@.mbtiles | sed 's/checkbuf/check/g' > $@.out
 	cmp $@.out $(patsubst %.checkbuf,%,$@)
 	rm $@.out $@.mbtiles

README.md

@@ -1,7 +1,7 @@
 tippecanoe
 ==========
 
-Builds [vector tilesets](https://www.mapbox.com/developers/vector-tiles/) from large (or small) collections of [GeoJSON](http://geojson.org/) features,
+Builds [vector tilesets](https://www.mapbox.com/developers/vector-tiles/) from large (or small) collections of [GeoJSON](http://geojson.org/) or [Geobuf](https://github.com/mapbox/geobuf) features,
 [like these](MADE_WITH.md).
 
 [![Build Status](https://travis-ci.org/mapbox/tippecanoe.svg)](https://travis-ci.org/mapbox/tippecanoe)
@@ -41,7 +41,7 @@ Usage
 -----
 
 ```sh
-$ tippecanoe -o file.mbtiles [file.json ...]
+$ tippecanoe -o file.mbtiles [file.json file.geobuf ...]
 ```
 
 If no files are specified, it reads GeoJSON from the standard input.
@@ -109,6 +109,7 @@ If your input is formatted as newline-delimited GeoJSON, use `-P` to make input
 ### Input files and layer names
 
  * _name_`.json` or _name_`.geojson`: Read the named GeoJSON input file into a layer called _name_.
+ * _name_`.geobuf` or _name_`.geobuf`: Read the named Geobuf input file into a layer called _name_.
  * `-l` _name_ or `--layer=`_name_: Use the specified layer name instead of deriving a name from the input filename or output tileset. If there are multiple input files
    specified, the files are all merged into the single named layer, even if they try to specify individual names with `-L`.
  * `-L` _name_`:`_file.json_ or `--named-layer=`_name_`:`_file.json_: Specify layer names for individual files. If your shell supports it, you can use a subshell redirect like `-L` _name_`:<(cat dir/*.json)` to specify a layer name for the output of streamed input.

geojson2nd.cpp

@@ -0,0 +1,153 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdarg.h>
+#include <unistd.h>
+#include <string>
+#include "jsonpull.h"
+
+int fail = EXIT_SUCCESS;
+bool wrap = false;
+
+std::string buffered;
+int buffered_type = -1;
+// 0: nothing yet
+// 1: buffered a line
+// 2: wrote the line and the wrapper
+int buffer_state = 0;
+
+void out(std::string s, int type) {
+	if (!wrap) {
+		printf("%s\n", s.c_str());
+		return;
+	}
+
+	if (buffer_state == 0) {
+		buffered = s;
+		buffered_type = type;
+		buffer_state = 1;
+		return;
+	}
+
+	if (buffer_state == 1) {
+		if (buffered_type == 1) {
+			printf("{\"type\":\"FeatureCollection\",\"features\":[\n");
+		} else {
+			printf("{\"type\":\"GeometryCollection\",\"geometries\":[\n");
+		}
+
+		printf("%s\n", buffered.c_str());
+		buffer_state = 2;
+	}
+
+	printf(",\n%s\n", s.c_str());
+
+	if (type != buffered_type) {
+		fprintf(stderr, "Error: mix of bare geometries and features\n");
+		exit(EXIT_FAILURE);
+	}
+}
+
+void process(FILE *fp, const char *fname) {
+	json_pull *jp = json_begin_file(fp);
+
+	while (1) {
+		json_object *j = json_read(jp);
+		if (j == NULL) {
+			if (jp->error != NULL) {
+				fprintf(stderr, "%s:%d: %s\n", fname, jp->line, jp->error);
+			}
+
+			json_free(jp->root);
+			break;
+		}
+
+		json_object *type = json_hash_get(j, "type");
+		if (type == NULL || type->type != JSON_STRING) {
+			continue;
+		}
+
+		if (strcmp(type->string, "Feature") == 0) {
+			char *s = json_stringify(j);
+			out(s, 1);
+			free(s);
+			json_free(j);
+		} else if (strcmp(type->string, "Point") == 0 ||
+			   strcmp(type->string, "MultiPoint") == 0 ||
+			   strcmp(type->string, "LineString") == 0 ||
+			   strcmp(type->string, "MultiLineString") == 0 ||
+			   strcmp(type->string, "MultiPolygon") == 0) {
+			int is_geometry = 1;
+
+			if (j->parent != NULL) {
+				if (j->parent->type == JSON_ARRAY) {
+					if (j->parent->parent->type == JSON_HASH) {
+						json_object *geometries = json_hash_get(j->parent->parent, "geometries");
+						if (geometries != NULL) {
+							// Parent of Parent must be a GeometryCollection
+							is_geometry = 0;
+						}
+					}
+				} else if (j->parent->type == JSON_HASH) {
+					json_object *geometry = json_hash_get(j->parent, "geometry");
+					if (geometry != NULL) {
+						// Parent must be a Feature
+						is_geometry = 0;
+					}
+				}
+			}
+
+			if (is_geometry) {
+				char *s = json_stringify(j);
+				out(s, 2);
+				free(s);
+				json_free(j);
+			}
+		} else if (strcmp(type->string, "FeatureCollection") == 0) {
+			json_free(j);
+		}
+	}
+
+	json_end(jp);
+}
+
+int main(int argc, char **argv) {
+	extern int optind;
+	int i;
+
+	while ((i = getopt(argc, argv, "w")) != -1) {
+		switch (i) {
+		case 'w':
+			wrap = true;
+			break;
+
+		default:
+			fprintf(stderr, "Unexpected option -%c\n", i);
+			exit(EXIT_FAILURE);
+		}
+	}
+
+	if (optind >= argc) {
+		process(stdin, "standard input");
+	} else {
+		for (i = optind; i < argc; i++) {
+			FILE *f = fopen(argv[i], "r");
+			if (f == NULL) {
+				perror(argv[i]);
+				exit(EXIT_FAILURE);
+			}
+
+			process(f, argv[i]);
+			fclose(f);
+		}
+	}
+
+	if (buffer_state == 1) {
+		printf("%s\n", buffered.c_str());
+	} else if (buffer_state == 2) {
+		printf("]}\n");
+	}
+
+	return fail;
+}

main.cpp

@@ -1193,7 +1193,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
 		}
 		size_t layer = a->second.id;
 
-		if (sources[source].file.size() > 4 && sources[source].file.substr(sources[source].file.size() - 4) == std::string(".pbf")) {
+		if (sources[source].file.size() > 7 && sources[source].file.substr(sources[source].file.size() - 7) == std::string(".geobuf")) {
 			struct stat st;
 			if (fstat(fd, &st) != 0) {
 				perror("fstat");

man/tippecanoe.1

@@ -1,6 +1,6 @@
 .TH tippecanoe
 .PP
-Builds vector tilesets \[la]https://www.mapbox.com/developers/vector-tiles/\[ra] from large (or small) collections of GeoJSON \[la]http://geojson.org/\[ra] features,
+Builds vector tilesets \[la]https://www.mapbox.com/developers/vector-tiles/\[ra] from large (or small) collections of GeoJSON \[la]http://geojson.org/\[ra] or Geobuf \[la]https://github.com/mapbox/geobuf\[ra] features,
 like these \[la]MADE_WITH.md\[ra]\&.
 .PP
 [Build Status](https://travis\-ci.org/mapbox/tippecanoe.svg) \[la]https://travis-ci.org/mapbox/tippecanoe\[ra]
@@ -37,7 +37,7 @@ $ brew install tippecanoe
 .PP
 .RS
 .nf
-$ tippecanoe \-o file.mbtiles [file.json ...]
+$ tippecanoe \-o file.mbtiles [file.json file.geobuf ...]
 .fi
 .RE
 .PP
@@ -112,6 +112,8 @@ or if metadata fields can't be set. You probably don't want to use this.
 .IP \(bu 2
 \fIname\fP\fB\fC\&.json\fR or \fIname\fP\fB\fC\&.geojson\fR: Read the named GeoJSON input file into a layer called \fIname\fP\&.
 .IP \(bu 2
+\fIname\fP\fB\fC\&.geobuf\fR or \fIname\fP\fB\fC\&.geobuf\fR: Read the named Geobuf input file into a layer called \fIname\fP\&.
+.IP \(bu 2
 \fB\fC\-l\fR \fIname\fP or \fB\fC\-\-layer=\fR\fIname\fP: Use the specified layer name instead of deriving a name from the input filename or output tileset. If there are multiple input files
 specified, the files are all merged into the single named layer, even if they try to specify individual names with \fB\fC\-L\fR\&.
 .IP \(bu 2

tests/geometry/onebare.json

@@ -0,0 +1 @@
+{ "type": "LineString", "coordinates": [ [ -122, 37 ], [ -118, 34 ] ] }

tests/geometry/out/-z3.json

@@ -1,9 +1,9 @@
 { "type": "FeatureCollection", "properties": {
-"bounds": "-79.453125,-23.563987,102.000000,59.900000",
+"bounds": "-122.000000,-23.563987,102.000000,59.900000",
 "center": "22.500000,20.489949,3",
 "description": "tests/geometry/out/-z3.json.check.mbtiles",
 "format": "pbf",
-"json": "{\"vector_layers\": [ { \"id\": \"bare\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {} }, { \"id\": \"geometrycollection\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {\"collection\": \"Boolean\"} }, { \"id\": \"multipoint\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {\"point\": \"String\"} } ],\"tilestats\": {\"layerCount\": 3,\"layers\": [{\"layer\": \"bare\",\"count\": 2,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []},{\"layer\": \"geometrycollection\",\"count\": 2,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"collection\",\"count\": 1,\"type\": \"boolean\",\"values\": [true]}]},{\"layer\": \"multipoint\",\"count\": 1,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"point\",\"count\": 1,\"type\": \"string\",\"values\": [\"multi\"]}]}]}}",
+"json": "{\"vector_layers\": [ { \"id\": \"bare\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {} }, { \"id\": \"geometrycollection\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {\"collection\": \"Boolean\"} }, { \"id\": \"multipoint\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {\"point\": \"String\"} }, { \"id\": \"onebare\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 3, \"fields\": {} } ],\"tilestats\": {\"layerCount\": 4,\"layers\": [{\"layer\": \"bare\",\"count\": 2,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []},{\"layer\": \"geometrycollection\",\"count\": 2,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"collection\",\"count\": 1,\"type\": \"boolean\",\"values\": [true]}]},{\"layer\": \"multipoint\",\"count\": 1,\"geometry\": \"Point\",\"attributeCount\": 1,\"attributes\": [{\"attribute\": \"point\",\"count\": 1,\"type\": \"string\",\"values\": [\"multi\"]}]},{\"layer\": \"onebare\",\"count\": 1,\"geometry\": \"LineString\",\"attributeCount\": 0,\"attributes\": []}]}}",
 "maxzoom": "3",
 "minzoom": "0",
 "name": "tests/geometry/out/-z3.json.check.mbtiles",
@@ -24,6 +24,10 @@
 { "type": "FeatureCollection", "properties": { "layer": "multipoint", "version": 2, "extent": 4096 }, "features": [
 { "type": "Feature", "properties": { "point": "multi" }, "geometry": { "type": "MultiPoint", "coordinates": [ [ -79.453125, 49.382373 ], [ 11.513672, 52.052490 ], [ -60.468750, -6.926427 ], [ 23.906250, -12.468760 ] ] } }
 ] }
+,
+{ "type": "FeatureCollection", "properties": { "layer": "onebare", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ -122.080078, 37.020098 ], [ -118.037109, 34.016242 ] ] } }
+] }
 ] }
 ,
 { "type": "FeatureCollection", "properties": { "zoom": 1, "x": 0, "y": 1 }, "features": [
@@ -42,6 +46,10 @@
 { "type": "FeatureCollection", "properties": { "layer": "multipoint", "version": 2, "extent": 4096 }, "features": [
 { "type": "Feature", "properties": { "point": "multi" }, "geometry": { "type": "Point", "coordinates": [ -79.453125, 49.382373 ] } }
 ] }
+,
+{ "type": "FeatureCollection", "properties": { "layer": "onebare", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ -122.036133, 37.020098 ], [ -118.037109, 34.016242 ] ] } }
+] }
 ] }
 ,
 { "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 1 }, "features": [
@@ -78,6 +86,12 @@
 ] }
 ] }
 ,
+{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 0, "y": 1 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "onebare", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ -122.014160, 37.002553 ], [ -118.015137, 34.016242 ] ] } }
+] }
+] }
+,
 { "type": "FeatureCollection", "properties": { "zoom": 2, "x": 1, "y": 2 }, "features": [
 { "type": "FeatureCollection", "properties": { "layer": "multipoint", "version": 2, "extent": 4096 }, "features": [
 { "type": "Feature", "properties": { "point": "multi" }, "geometry": { "type": "Point", "coordinates": [ -60.468750, -6.991859 ] } }
@@ -132,6 +146,12 @@
 ] }
 ] }
 ,
+{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 1, "y": 3 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "onebare", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "LineString", "coordinates": [ [ -122.003174, 37.002553 ], [ -118.004150, 34.007135 ] ] } }
+] }
+] }
+,
 { "type": "FeatureCollection", "properties": { "zoom": 3, "x": 2, "y": 4 }, "features": [
 { "type": "FeatureCollection", "properties": { "layer": "multipoint", "version": 2, "extent": 4096 }, "features": [
 { "type": "Feature", "properties": { "point": "multi" }, "geometry": { "type": "Point", "coordinates": [ -60.468750, -7.002764 ] } }