diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-06 10:25:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #164 from mapbox/parallel-test

Browse Source 
Spell out a fuzz test for the four input-reading code paths
This commit is contained in:
Eric Fischer
2016-02-02 14:14:26 -08:00
parent f5ea5974ee 59c5d51e25
commit 49324a3dc6
2 changed files with 33 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+17 -1
View File
@@ -66,7 +66,7 @@ indent:
TESTS = $(wildcard tests/*/out/*.json) TESTS = $(wildcard tests/*/out/*.json)
SPACE = $(NULL) $(NULL) SPACE = $(NULL) $(NULL)
test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) parallel-test
%.json.check: %.json.check:
./tippecanoe -f -o $@.mbtiles $(subst _, ,$(patsubst %.json.check,%,$(word 4,$(subst /, ,$@)))) $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json) ./tippecanoe -f -o $@.mbtiles $(subst _, ,$(patsubst %.json.check,%,$(word 4,$(subst /, ,$@)))) $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json)
@@ -74,6 +74,22 @@ test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS))
cmp $(patsubst %.check,%,$@) $@.out cmp $(patsubst %.check,%,$@) $@.out
rm $@.out $@.mbtiles rm $@.out $@.mbtiles
parallel-test:
mkdir -p tests/parallel
perl -e 'for ($$i = 0; $$i < 300000; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in.json
./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-file.mbtiles tests/parallel/in.json
./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-file.mbtiles tests/parallel/in.json
cat tests/parallel/in.json | ./tippecanoe -z5 -f -pi -l test -n test -o tests/parallel/linear-pipe.mbtiles
cat tests/parallel/in.json | ./tippecanoe -z5 -f -pi -l test -n test -P -o tests/parallel/parallel-pipe.mbtiles
./tippecanoe-decode tests/parallel/linear-file.mbtiles > tests/parallel/linear-file.json
./tippecanoe-decode tests/parallel/parallel-file.mbtiles > tests/parallel/parallel-file.json
./tippecanoe-decode tests/parallel/linear-pipe.mbtiles > tests/parallel/linear-pipe.json
./tippecanoe-decode tests/parallel/parallel-pipe.mbtiles > tests/parallel/parallel-pipe.json
cmp tests/parallel/linear-file.json tests/parallel/parallel-file.json
cmp tests/parallel/linear-file.json tests/parallel/linear-pipe.json
cmp tests/parallel/linear-file.json tests/parallel/parallel-pipe.json
rm tests/parallel/*.mbtiles tests/parallel/*.json
# Use this target to regenerate the standards that the tests are compared against # Use this target to regenerate the standards that the tests are compared against
# after making a change that legitimately changes their output # after making a change that legitimately changes their output
geojson.c
+12 -4
View File
@@ -197,10 +197,18 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL
} }
if (!*initialized) { if (!*initialized) {
if (x < 0 || x >= (1LL << 32) || y < 0 || y >= (1LL < 32)) {
*initial_x = 1LL << 31;
*initial_y = 1LL << 31;
*wx = 1LL << 31;
*wy = 1LL << 31;
} else {
*initial_x = (x >> geometry_scale) << geometry_scale; *initial_x = (x >> geometry_scale) << geometry_scale;
*initial_y = (y >> geometry_scale) << geometry_scale; *initial_y = (y >> geometry_scale) << geometry_scale;
*wx = x; *wx = x;
*wy = y; *wy = y;
}
*initialized = 1; *initialized = 1;
} }
@@ -451,7 +459,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, char *s, c
return off; return off;
} }
int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y) { int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y) {
json_object *geometry_type = json_hash_get(geometry, "type"); json_object *geometry_type = json_hash_get(geometry, "type");
if (geometry_type == NULL) { if (geometry_type == NULL) {
static int warned = 0; static int warned = 0;
@@ -652,7 +660,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha
return 1; return 1;
} }
void parse_json(json_pull *jp, const char *reading, long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y) { void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, struct pool *exclude, struct pool *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int maxzoom, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y) {
long long found_hashes = 0; long long found_hashes = 0;
long long found_features = 0; long long found_features = 0;
long long found_geometries = 0; long long found_geometries = 0;
@@ -767,7 +775,7 @@ void parse_json(json_pull *jp, const char *reading, long long *layer_seq, volati
struct parse_json_args { struct parse_json_args {
json_pull *jp; json_pull *jp;
const char *reading; const char *reading;
long long *layer_seq; volatile long long *layer_seq;
volatile long long *progress_seq; volatile long long *progress_seq;
long long *metapos; long long *metapos;
long long *geompos; long long *geompos;
@@ -932,7 +940,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
} }
} }
long long layer_seq[CPUS]; volatile long long layer_seq[CPUS];
for (i = 0; i < CPUS; i++) { for (i = 0; i < CPUS; i++) {
// To preserve feature ordering, unique id for each segment // To preserve feature ordering, unique id for each segment
// begins with that segment's offset into the input // begins with that segment's offset into the input