From dfaba5a3e644d71f81bad917c1e97b6967f31d27 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 2 Feb 2016 12:21:19 -0800 Subject: [PATCH 1/3] Spell out a fuzz test for the four input-reading code paths --- Makefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Makefile b/Makefile index 1887849a..c30bace9 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,22 @@ test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) cmp $(patsubst %.check,%,$@) $@.out rm $@.out $@.mbtiles +parallel-test: + mkdir -p tests/parallel + perl -e 'for ($$i = 0; $$i < 300000; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in.json + ./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-file.mbtiles tests/parallel/in.json + ./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-file.mbtiles tests/parallel/in.json + cat tests/parallel/in.json | ./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-pipe.mbtiles + cat tests/parallel/in.json | ./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-pipe.mbtiles + ./tippecanoe-decode tests/parallel/linear-file.mbtiles > tests/parallel/linear-file.json + ./tippecanoe-decode tests/parallel/parallel-file.mbtiles > tests/parallel/parallel-file.json + ./tippecanoe-decode tests/parallel/linear-pipe.mbtiles > tests/parallel/linear-pipe.json + ./tippecanoe-decode tests/parallel/parallel-pipe.mbtiles > tests/parallel/parallel-pipe.json + cmp tests/parallel/linear-file.json tests/parallel/parallel-file.json + cmp tests/parallel/linear-file.json tests/parallel/linear-pipe.json + cmp tests/parallel/linear-file.json tests/parallel/parallel-pipe.json + rm tests/parallel/*.mbtiles tests/parallel/*.json + # Use this target to regenerate the standards that the tests are compared against # after making a change that legitimately changes their output From 54d9e20e3048439d36398ab3b68d90b42a34faca Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 2 Feb 2016 12:23:30 -0800 Subject: [PATCH 2/3] Add the input-paths test to the standard test target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c30bace9..3d68b491 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ indent: TESTS = $(wildcard tests/*/out/*.json) SPACE = $(NULL) $(NULL) -test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) +test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) parallel-test %.json.check: ./tippecanoe -f -o $@.mbtiles $(subst _, ,$(patsubst %.json.check,%,$(word 4,$(subst /, ,$@)))) $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json) From 59c5d51e250347da07c204190bdc23eec759f413 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 2 Feb 2016 13:33:49 -0800 Subject: [PATCH 3/3] Don't allow the initial coordinate pair to be out of bounds --- geojson.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/geojson.c b/geojson.c index 4a028197..5e4edd61 100644 --- a/geojson.c +++ b/geojson.c @@ -197,10 +197,18 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL } if (!*initialized) { - *initial_x = (x >> geometry_scale) << geometry_scale; - *initial_y = (y >> geometry_scale) << geometry_scale; - *wx = x; - *wy = y; + if (x < 0 || x >= (1LL << 32) || y < 0 || y >= (1LL < 32)) { + *initial_x = 1LL << 31; + *initial_y = 1LL << 31; + *wx = 1LL << 31; + *wy = 1LL << 31; + } else { + *initial_x = (x >> geometry_scale) << geometry_scale; + *initial_y = (y >> geometry_scale) << geometry_scale; + *wx = x; + *wy = y; + } + *initialized = 1; } @@ -451,7 +459,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, char *s, c return off; } -int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y) { +int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y) { json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == NULL) { static int warned = 0; @@ -652,7 +660,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha return 1; } -void parse_json(json_pull *jp, const char *reading, long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y) { +void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; @@ -767,7 +775,7 @@ void parse_json(json_pull *jp, const char *reading, long long *layer_seq, volati struct parse_json_args { json_pull *jp; const char *reading; - long long *layer_seq; + volatile long long *layer_seq; volatile long long *progress_seq; long long *metapos; long long *geompos; @@ -932,7 +940,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const } } - long long layer_seq[CPUS]; + volatile long long layer_seq[CPUS]; for (i = 0; i < CPUS; i++) { // To preserve feature ordering, unique id for each segment // begins with that segment's offset into the input