diff --git a/CHANGELOG.md b/CHANGELOG.md index b6f5ec49..1af2b461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,37 @@ +# 2.82.0 + +* Fix corruption of a JSON array when a non-final element was removed from it. + The old `json_free` / `json_disconnect` passed an element count to `memmove` + where a byte count was required, so pruning element 0 of an 8-element array + left the first two slots pointing at the same node -- a double free at + teardown -- and silently dropped the last element. Only reachable through a + whole-document tree, because removing the most recently added element made + the bad `memmove` a zero-length no-op. (#388) +* Rewrite `jsonpull` in C++ with `unique_ptr` ownership, `std::vector` for + arrays and hash entries, and `std::string` for string values, replacing the + hand-rolled `malloc`/`realloc`/`free` memory management. Value payloads now + live in type-tagged subclasses reached through asserting accessors, so + reading a hash as a string fails immediately instead of silently returning + garbage. (#388) +* Fix `tippecanoe-json-tool --extract` crashing on a numeric attribute, which + read the number's storage as a string pointer. (#388) +* Fix decoding of a `\u` escape sequence in which a high surrogate is followed + by a non-surrogate BMP code point, which combined the two into a single + wrong code point. (#388) +* Fix a `\uFFFF` escape being decoded to the overlong, invalid four-byte + UTF-8 sequence `F0 8F BF BF` instead of `EF BF BF`. (#388) +* Fix tile-join reading a non-string field type out of a tileset's tilejson. + (#388) +* Fix `tippecanoe-decode` and tile-join crashing on a directory tileset whose + `metadata.json` holds a non-string value, such as a numeric `minzoom` or a + nested object. Those entries are now reported and skipped. (#388) +* Fix an uninitialized read when a prefilter or postfilter emitted a feature + with `"properties": null`. Both filter readers accepted a null `properties` + and then read its length as though it were a hash, which was never + initialized for a non-container node. (#388) +* Fix the include guard in `evaluator.hpp`, which defined `EVALUATOR HPP` + instead of `EVALUATOR_HPP` and so never guarded anything. (#388) + # 2.81.0 * Add `--drop-by-attribute-as-needed=`*attribute* to drop the features with diff --git a/Makefile b/Makefile index 69a1625a..3433af0a 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,7 @@ clean: rm -f ./tippecanoe ./tippecanoe-* ./tile-join ./unit *.o *.d */*.o */*.d tests/**/*.mbtiles tests/**/*.check indent: - clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false, SortIncludes: false}" $(filter-out flatgeobuf.cpp,$(C)) $(H) jsonpull/*.[ch] + clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false, SortIncludes: false}" $(filter-out flatgeobuf.cpp,$(C)) $(H) jsonpull/jsonpull.h jsonpull/jsonpull.cpp TESTS = $(wildcard tests/*/out/*.json) SPACE = $(NULL) $(NULL) @@ -261,6 +261,14 @@ raw-tiles-test: tippecanoe tippecanoe-decode tile-join ./tippecanoe-decode -x generator tests/raw-tiles/nothing > tests/raw-tiles/nothing.json.check cmp tests/raw-tiles/nothing.json.check tests/raw-tiles/nothing.json rm -r tests/raw-tiles/nothing tests/raw-tiles/nothing.json.check + # Test that a non-string value in metadata.json is reported and skipped + # instead of being read as a string (which used to crash) + ./tippecanoe -q -f -e tests/raw-tiles/nonstring tests/raw-tiles/hackspots.geojson + sed -i.bak 's/"minzoom": "0"/"minzoom": 0/' tests/raw-tiles/nonstring/metadata.json + rm tests/raw-tiles/nonstring/metadata.json.bak + grep -q '"minzoom": 0' tests/raw-tiles/nonstring/metadata.json + ./tippecanoe-decode -x generator tests/raw-tiles/nonstring > /dev/null + rm -r tests/raw-tiles/nonstring pmtiles-test: tippecanoe tippecanoe-decode tile-join ./tippecanoe -q -f -o tests/pmtiles/hackspots.pmtiles -r1 -pC tests/raw-tiles/hackspots.geojson diff --git a/attribute.cpp b/attribute.cpp index 53c9dae5..71f0d1cd 100644 --- a/attribute.cpp +++ b/attribute.cpp @@ -42,10 +42,10 @@ void set_attribute_accum(std::unordered_map &attribut void set_attribute_accum(std::unordered_map &attribute_accum, const char *arg, char **argv) { if (*arg == '{') { - json_pull *jp = json_begin_string(arg); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(arg); + json_object_ptr o = json_read_tree(jp); - if (o == NULL) { + if (o == nullptr) { fprintf(stderr, "%s: -E%s: %s\n", *argv, arg, jp->error); exit(EXIT_JSON); } @@ -55,24 +55,21 @@ void set_attribute_accum(std::unordered_map &attribut exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.length; i++) { - json_object *k = o->value.object.keys[i]; - json_object *v = o->value.object.values[i]; + for (size_t i = 0; i < o->entries().size(); i++) { + const auto &e = o->entries()[i]; - if (k->type != JSON_STRING) { + if (e.key->type != JSON_STRING) { fprintf(stderr, "%s: -E%s: key %zu not a string\n", *argv, arg, i); exit(EXIT_JSON); } - if (v->type != JSON_STRING) { + if (e.value->type != JSON_STRING) { fprintf(stderr, "%s: -E%s: value %zu not a string\n", *argv, arg, i); exit(EXIT_JSON); } - set_attribute_accum(attribute_accum, k->value.string.string, v->value.string.string); + set_attribute_accum(attribute_accum, e.key->string(), e.value->string()); } - json_free(o); - json_end(jp); return; } diff --git a/dirtiles.cpp b/dirtiles.cpp index 98138bd5..b7dd3945 100644 --- a/dirtiles.cpp +++ b/dirtiles.cpp @@ -248,9 +248,9 @@ sqlite3 *dirmeta2tmp(const char *fname) { if (f == NULL) { perror(name.c_str()); } else { - json_pull *jp = json_begin_file(f); - json_object *o = json_read_tree(jp); - if (o == NULL) { + json_pull_ptr jp = json_begin_file(f); + json_object_ptr o = json_read_tree(jp); + if (o == nullptr) { fprintf(stderr, "%s: metadata parsing error: %s\n", name.c_str(), jp->error); exit(EXIT_JSON); } @@ -260,19 +260,22 @@ sqlite3 *dirmeta2tmp(const char *fname) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.length; i++) { - if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) { + for (const auto &e : o->entries()) { + // Skip, not just warn: a metadata.json from another tool may + // have a numeric minzoom or a nested object, and string() + // asserts on the type. + if (e.key->type != JSON_STRING || e.value->type != JSON_STRING) { fprintf(stderr, "%s: non-string in metadata\n", name.c_str()); + continue; } - char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->value.object.keys[i]->value.string.string, o->value.object.values[i]->value.string.string); + char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", e.key->string().c_str(), e.value->string().c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", o->value.object.keys[i]->value.string.string, err); + fprintf(stderr, "set %s in metadata: %s\n", e.key->string().c_str(), err); } sqlite3_free(sql); } - json_end(jp); fclose(f); } diff --git a/evaluator.cpp b/evaluator.cpp index 38566ae2..ce1e576c 100644 --- a/evaluator.cpp +++ b/evaluator.cpp @@ -17,7 +17,7 @@ int compare(mvt_value const &one, json_object *two, bool &fail) { return false; // string vs non-string } - return strcmp(one.c_str(), two->value.string.string); + return strcmp(one.c_str(), two->string().c_str()); case mvt_double: case mvt_float: @@ -52,9 +52,9 @@ int compare(mvt_value const &one, json_object *two, bool &fail) { exit(EXIT_IMPOSSIBLE); } - if (v < two->value.number.number) { + if (v < two->number()) { return -1; - } else if (v > two->value.number.number) { + } else if (v > two->number()) { return 1; } else { return 0; @@ -92,7 +92,7 @@ int compare(mvt_value const &one, json_object *two, bool &fail) { // 1: true // -1: incomparable (sql null), treated as false in final output static int eval(std::function feature, json_object *f, std::set &exclude_attributes, std::vector const &unidecode_data) { - if (f != NULL) { + if (f != nullptr) { if (f->type == JSON_TRUE) { return 1; } else if (f->type == JSON_FALSE) { @@ -102,7 +102,7 @@ static int eval(std::function feature, json_obje } if (f->type == JSON_NUMBER) { - if (f->value.number.number == 0) { + if (f->number() == 0) { return 0; } else { return 1; @@ -110,7 +110,7 @@ static int eval(std::function feature, json_obje } if (f->type == JSON_STRING) { - if (f->value.string.string[0] == '\0') { + if (f->string().empty()) { return 0; } else { return 1; @@ -118,131 +118,129 @@ static int eval(std::function feature, json_obje } } - if (f == NULL || f->type != JSON_ARRAY) { - fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f)); + if (f == nullptr || f->type != JSON_ARRAY) { + fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.length < 1) { - fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f)); + if (f->array().size() < 1) { + fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[0]->type != JSON_STRING) { - fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f)); + if (f->array()[0]->type != JSON_STRING) { + fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (strcmp(f->value.array.array[0]->value.string.string, "has") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) { - if (f->value.array.length != 2) { - fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f)); + const std::string &op = f->array()[0]->string(); + + if (op == "has" || + op == "!has") { + if (f->array().size() != 2) { + fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (strcmp(f->value.array.array[0]->value.string.string, "has") == 0) { - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f)); + if (op == "has") { + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - return feature(std::string(f->value.array.array[1]->value.string.string)).type != mvt_no_such_key; + return feature(f->array()[1]->string()).type != mvt_no_such_key; } - if (strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) { - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f)); + if (op == "!has") { + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - return feature(std::string(f->value.array.array[1]->value.string.string)).type == mvt_no_such_key; + return feature(f->array()[1]->string()).type == mvt_no_such_key; } } - if (strcmp(f->value.array.array[0]->value.string.string, "==") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "!=") == 0 || - strcmp(f->value.array.array[0]->value.string.string, ">") == 0 || - strcmp(f->value.array.array[0]->value.string.string, ">=") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "<") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "<=") == 0) { - if (f->value.array.length != 3) { - fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f)); + if (op == "==" || + op == "!=" || + op == ">" || + op == ">=" || + op == "<" || + op == "<=") { + if (f->array().size() != 3) { + fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "comparison key is not a string: %s\n", json_stringify(f)); + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "comparison key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string)); + mvt_value ff = feature(f->array()[1]->string()); if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: attribute not found for comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: attribute not found for comparison: %s\n", json_stringify(f).c_str()); warned = true; } - if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) { + if (op == "!=") { return true; // attributes that aren't found are not equal } return false; // not found: comparison is false } bool fail = false; - int cmp = compare(ff, f->value.array.array[2], fail); + int cmp = compare(ff, f->array()[2].get(), fail); if (fail) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: mismatched type in comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: mismatched type in comparison: %s\n", json_stringify(f).c_str()); warned = true; } - if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) { + if (op == "!=") { return true; // mismatched types are not equal } return false; } - if (strcmp(f->value.array.array[0]->value.string.string, "==") == 0) { + if (op == "==") { return cmp == 0; } - if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) { + if (op == "!=") { return cmp != 0; } - if (strcmp(f->value.array.array[0]->value.string.string, ">") == 0) { + if (op == ">") { return cmp > 0; } - if (strcmp(f->value.array.array[0]->value.string.string, ">=") == 0) { + if (op == ">=") { return cmp >= 0; } - if (strcmp(f->value.array.array[0]->value.string.string, "<") == 0) { + if (op == "<") { return cmp < 0; } - if (strcmp(f->value.array.array[0]->value.string.string, "<=") == 0) { + if (op == "<=") { return cmp <= 0; } - fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f)); + fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f).c_str()); exit(EXIT_IMPOSSIBLE); } - if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "any") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "none") == 0) { + if (op == "all" || + op == "any" || + op == "none") { bool v; - if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) { + if (op == "all") { v = true; } else { v = false; } - for (size_t i = 1; i < f->value.array.length; i++) { - int out = eval(feature, f->value.array.array[i], exclude_attributes, unidecode_data); + for (size_t i = 1; i < f->array().size(); i++) { + int out = eval(feature, f->array()[i].get(), exclude_attributes, unidecode_data); if (out >= 0) { // nulls are ignored in boolean and/or expressions - if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) { + if (op == "all") { v = v && out; if (!v) { break; @@ -256,51 +254,47 @@ static int eval(std::function feature, json_obje } } - if (strcmp(f->value.array.array[0]->value.string.string, "none") == 0) { + if (op == "none") { return !v; } else { return v; } } - if (strcmp(f->value.array.array[0]->value.string.string, "in") == 0 || - strcmp(f->value.array.array[0]->value.string.string, "!in") == 0) { - if (f->value.array.length < 2) { - fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f)); + if (op == "in" || + op == "!in") { + if (f->array().size() < 2) { + fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"!in\" key is not a string: %s\n", json_stringify(f)); + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "\"!in\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string)); + mvt_value ff = feature(f->array()[1]->string()); if (ff.type == mvt_no_such_key) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: attribute not found for comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: attribute not found for comparison: %s\n", json_stringify(f).c_str()); warned = true; } - if (strcmp(f->value.array.array[0]->value.string.string, "!in") == 0) { + if (op == "!in") { return true; // attributes that aren't found are not in } return false; // not found: comparison is false } bool found = false; - for (size_t i = 2; i < f->value.array.length; i++) { + for (size_t i = 2; i < f->array().size(); i++) { bool fail = false; - int cmp = compare(ff, f->value.array.array[i], fail); + int cmp = compare(ff, f->array()[i].get(), fail); if (fail) { static bool warned = false; if (!warned) { - const char *s = json_stringify(f); - fprintf(stderr, "Warning: mismatched type in comparison: %s\n", s); - free((void *) s); + fprintf(stderr, "Warning: mismatched type in comparison: %s\n", json_stringify(f).c_str()); warned = true; } cmp = 1; @@ -312,39 +306,39 @@ static int eval(std::function feature, json_obje } } - if (strcmp(f->value.array.array[0]->value.string.string, "in") == 0) { + if (op == "in") { return found; } else { return !found; } } - if (strcmp(f->value.array.array[0]->value.string.string, "attribute-filter") == 0) { - if (f->value.array.length != 3) { - fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f)); + if (op == "attribute-filter") { + if (f->array().size() != 3) { + fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - if (f->value.array.array[1]->type != JSON_STRING) { - fprintf(stderr, "\"attribute-filter\" key is not a string: %s\n", json_stringify(f)); + if (f->array()[1]->type != JSON_STRING) { + fprintf(stderr, "\"attribute-filter\" key is not a string: %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } - bool ok = eval(feature, f->value.array.array[2], exclude_attributes, unidecode_data) > 0; + bool ok = eval(feature, f->array()[2].get(), exclude_attributes, unidecode_data) > 0; if (!ok) { - exclude_attributes.insert(f->value.array.array[1]->value.string.string); + exclude_attributes.insert(f->array()[1]->string()); } return true; } - fprintf(stderr, "Unknown filter %s\n", json_stringify(f)); + fprintf(stderr, "Unknown filter %s\n", json_stringify(f).c_str()); exit(EXIT_FILTER); } -bool evaluate(std::function feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data) { - if (filter == NULL || filter->type != JSON_HASH) { - fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter)); +static bool evaluate(std::function feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data) { + if (filter == nullptr || filter->type != JSON_HASH) { + fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter).c_str()); exit(EXIT_JSON); } @@ -352,47 +346,43 @@ bool evaluate(std::function feature, std::string json_object *f; f = json_hash_get(filter, layer.c_str()); - if (ok && f != NULL) { + if (ok && f != nullptr) { ok = eval(feature, f, exclude_attributes, unidecode_data) > 0; } f = json_hash_get(filter, "*"); - if (ok && f != NULL) { + if (ok && f != nullptr) { ok = eval(feature, f, exclude_attributes, unidecode_data) > 0; } return ok; } -json_object *read_filter(const char *fname) { +json_object_ptr read_filter(const char *fname) { FILE *fp = fopen(fname, "r"); if (fp == NULL) { perror(fname); exit(EXIT_OPEN); } - json_pull *jp = json_begin_file(fp); - json_object *filter = json_read_tree(jp); - if (filter == NULL) { + json_pull_ptr jp = json_begin_file(fp); + json_object_ptr filter = json_read_tree(jp); + if (filter == nullptr) { fprintf(stderr, "%s: %s\n", fname, jp->error); exit(EXIT_JSON); } - json_disconnect(filter); - json_end(jp); fclose(fp); return filter; } -json_object *parse_filter(const char *s) { - json_pull *jp = json_begin_string(s); - json_object *filter = json_read_tree(jp); - if (filter == NULL) { +json_object_ptr parse_filter(const char *s) { + json_pull_ptr jp = json_begin_string(s); + json_object_ptr filter = json_read_tree(jp); + if (filter == nullptr) { fprintf(stderr, "Could not parse filter %s\n", s); fprintf(stderr, "%s\n", jp->error); exit(EXIT_JSON); } - json_disconnect(filter); - json_end(jp); return filter; } diff --git a/evaluator.hpp b/evaluator.hpp index bc6be00c..4be1582d 100644 --- a/evaluator.hpp +++ b/evaluator.hpp @@ -1,5 +1,5 @@ #ifndef EVALUATOR_HPP -#define EVALUATOR HPP +#define EVALUATOR_HPP #include #include @@ -7,9 +7,13 @@ #include "jsonpull/jsonpull.h" #include "mvt.hpp" +// The `filter` parameters take a borrowed pointer; the caller (in +// main.cpp, tile-join, overzoom) keeps the json_object_ptr alive +// across every per-feature evaluate() call. A raw pointer avoids +// touching unique_ptr at all on this hot path. bool evaluate(std::unordered_map const &feature, std::string const &layer, json_object *filter, std::set &exclude_attributes, std::vector const &unidecode_data); -json_object *parse_filter(const char *s); -json_object *read_filter(const char *fname); +json_object_ptr parse_filter(const char *s); +json_object_ptr read_filter(const char *fname); bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set &exclude_attributes, int z, std::vector const &unidecode_data); diff --git a/geobuf.cpp b/geobuf.cpp index 02d8ded3..6481695e 100644 --- a/geobuf.cpp +++ b/geobuf.cpp @@ -394,28 +394,25 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vectorsecond.s.c_str()); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(tip->second.s.c_str()); + json_object_ptr o = json_read_tree(jp); - if (o != NULL) { + if (o != nullptr) { json_object *min = json_hash_get(o, "minzoom"); - if (min != NULL && (min->type == JSON_NUMBER)) { - sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number)); + if (min != nullptr && (min->type == JSON_NUMBER)) { + sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number())); } json_object *max = json_hash_get(o, "maxzoom"); - if (max != NULL && (max->type == JSON_NUMBER)) { - sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number)); + if (max != nullptr && (max->type == JSON_NUMBER)) { + sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number())); } json_object *tlayer = json_hash_get(o, "layer"); - if (tlayer != NULL && (tlayer->type == JSON_STRING)) { - layername = tlayer->value.string.string; + if (tlayer != nullptr && (tlayer->type == JSON_STRING)) { + layername = tlayer->string(); } } - - json_free(o); - json_end(jp); } serialize_feature(sst, sf, layername); diff --git a/geojson-loop.cpp b/geojson-loop.cpp index 199c6531..e4f2f40b 100644 --- a/geojson-loop.cpp +++ b/geojson-loop.cpp @@ -26,34 +26,34 @@ static const char *geometry_names[GEOM_TYPES] = { // XXX duplicated static void json_context(json_object *j) { - char *s = json_stringify(j); + std::string s = json_stringify(j); - if (strlen(s) >= 500) { - snprintf(s + 497, strlen(s) + 1 - 497, "..."); + if (s.size() >= 500) { + s.resize(497); + s.append("..."); } - fprintf(stderr, "in JSON object %s\n", s); - free(s); // stringify + fprintf(stderr, "in JSON object %s\n", s.c_str()); } -void parse_json(json_feature_action *jfa, json_pull *jp) { +void parse_json(json_feature_action *jfa, json_pull_ptr &jp) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; while (1) { json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { + if (j == nullptr) { + if (jp->error != nullptr) { fprintf(stderr, "%s:%d: %s: ", jfa->fname.c_str(), jp->line, jp->error); - if (jp->root != NULL) { - json_context(jp->root); + if (jp->root != nullptr) { + json_context(jp->root.get()); } else { fprintf(stderr, "\n"); } } - json_free(jp->root); + jp->root.reset(); break; } @@ -66,7 +66,7 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { + if (type == nullptr || type->type != JSON_STRING) { continue; } @@ -74,25 +74,25 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { int i; int is_geometry = 0; for (i = 0; i < GEOM_TYPES; i++) { - if (strcmp(type->value.string.string, geometry_names[i]) == 0) { + if (type->string() == geometry_names[i]) { is_geometry = 1; break; } } if (is_geometry) { - if (j->parent != NULL) { - if (j->parent->type == JSON_ARRAY && j->parent->parent != NULL) { + if (j->parent != nullptr) { + if (j->parent->type == JSON_ARRAY && j->parent->parent != nullptr) { if (j->parent->parent->type == JSON_HASH) { json_object *geometries = json_hash_get(j->parent->parent, "geometries"); - if (geometries != NULL) { + if (geometries != nullptr) { // Parent of Parent must be a GeometryCollection is_geometry = 0; } } } else if (j->parent->type == JSON_HASH) { json_object *geometry = json_hash_get(j->parent, "geometry"); - if (geometry != NULL) { + if (geometry != nullptr) { // Parent must be a Feature is_geometry = 0; } @@ -102,8 +102,8 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { if (is_geometry) { json_object *jo = j; - while (jo != NULL) { - if (jo->parent != NULL && jo->parent->type == JSON_HASH) { + while (jo != nullptr) { + if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { if (json_hash_get(jo->parent, "properties") == jo) { // Ancestor is the value corresponding to a properties key is_geometry = 0; @@ -120,14 +120,14 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } found_geometries++; - jfa->add_feature(j, false, NULL, NULL, NULL, j); + jfa->add_feature(j, false, nullptr, nullptr, nullptr, j); json_free(j); continue; } } - if (strcmp(type->value.string.string, "Feature") != 0) { - if (strcmp(type->value.string.string, "FeatureCollection") == 0) { + if (type->string() != "Feature") { + if (type->string() == "FeatureCollection") { jfa->check_crs(j); json_free(j); } @@ -141,7 +141,7 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { found_features++; json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { + if (geometry == nullptr) { fprintf(stderr, "%s:%d: feature with no geometry: ", jfa->fname.c_str(), jp->line); json_context(j); json_free(j); @@ -149,7 +149,7 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { } json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "%s:%d: feature without properties hash: ", jfa->fname.c_str(), jp->line); json_context(j); json_free(j); @@ -159,8 +159,8 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { bool is_feature = true; { json_object *jo = j; - while (jo != NULL) { - if (jo->parent != NULL && jo->parent->type == JSON_HASH) { + while (jo != nullptr) { + if (jo->parent != nullptr && jo->parent->type == JSON_HASH) { if (json_hash_get(jo->parent, "properties") == jo) { // Ancestor is the value corresponding to a properties key is_feature = false; @@ -178,7 +178,7 @@ void parse_json(json_feature_action *jfa, json_pull *jp) { json_object *id = json_hash_get(j, "id"); json_object *geometries = json_hash_get(geometry, "geometries"); - if (geometries != NULL && geometries->type == JSON_ARRAY) { + if (geometries != nullptr && geometries->type == JSON_ARRAY) { jfa->add_feature(geometries, true, properties, id, tippecanoe, j); } else { jfa->add_feature(geometry, false, properties, id, tippecanoe, j); diff --git a/geojson-loop.hpp b/geojson-loop.hpp index 3d82be8e..acdb43d7 100644 --- a/geojson-loop.hpp +++ b/geojson-loop.hpp @@ -8,4 +8,4 @@ struct json_feature_action { virtual void check_crs(json_object *j) = 0; }; -void parse_json(json_feature_action *action, json_pull *jp); +void parse_json(json_feature_action *action, json_pull_ptr &jp); diff --git a/geojson.cpp b/geojson.cpp index 3798d14b..d9bc569f 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -42,7 +42,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string const &layername) { json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { + if (geometry_type == nullptr) { static int warned = 0; if (!warned) { fprintf(stderr, "%s:%d: null geometry (additional not reported): ", sst->fname, sst->line); @@ -60,7 +60,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom } json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: feature without coordinates array: ", sst->fname, sst->line); json_context(feature); return 0; @@ -68,12 +68,12 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom int t; for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) { + if (geometry_type->string() == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->value.string.string); + fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->string().c_str()); json_context(feature); return 0; } @@ -82,48 +82,48 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom int tippecanoe_maxzoom = -1; std::string tippecanoe_layername = layername; - if (tippecanoe != NULL) { + if (tippecanoe != nullptr) { json_object *min = json_hash_get(tippecanoe, "minzoom"); - if (min != NULL && (min->type == JSON_NUMBER)) { - tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number)); + if (min != nullptr && (min->type == JSON_NUMBER)) { + tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number())); } json_object *max = json_hash_get(tippecanoe, "maxzoom"); - if (max != NULL && (max->type == JSON_NUMBER)) { - tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number)); + if (max != nullptr && (max->type == JSON_NUMBER)) { + tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number())); } json_object *ln = json_hash_get(tippecanoe, "layer"); - if (ln != NULL && (ln->type == JSON_STRING)) { - tippecanoe_layername = std::string(ln->value.string.string); + if (ln != nullptr && (ln->type == JSON_STRING)) { + tippecanoe_layername = ln->string(); } } bool has_id = false; unsigned long long id_value = 0; - if (id != NULL) { + if (id != nullptr) { if (id->type == JSON_NUMBER) { - if (id->value.number.number >= 0) { + if (id->number() >= 0) { char *err = NULL; - std::string id_number = milo::dtoa_milo(id->value.number.number); + std::string id_number = milo::dtoa_milo(id->number()); id_value = strtoull(id_number.c_str(), &err, 10); - if (id->value.number.large_unsigned != 0) { - id_value = id->value.number.large_unsigned; + if (id->large_unsigned() != 0) { + id_value = id->large_unsigned(); } if (err != NULL && *err != '\0') { static bool warned_frac = false; if (!warned_frac) { - fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str()); + fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", milo::dtoa_milo(id->number()).c_str()); warned_frac = true; } - } else if (id->value.number.large_unsigned == 0 && std::to_string(id_value) != milo::dtoa_milo(id->value.number.number)) { + } else if (id->large_unsigned() == 0 && std::to_string(id_value) != milo::dtoa_milo(id->number())) { static bool warned = false; if (!warned) { - fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str()); + fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", milo::dtoa_milo(id->number()).c_str()); warned = true; } } else { @@ -133,7 +133,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom static bool warned_neg = false; if (!warned_neg) { - fprintf(stderr, "Warning: Can't represent negative feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str()); + fprintf(stderr, "Warning: Can't represent negative feature ID %s\n", milo::dtoa_milo(id->number()).c_str()); warned_neg = true; } } @@ -142,20 +142,20 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom if (additional[A_CONVERT_NUMERIC_IDS] && id->type == JSON_STRING) { char *err = NULL; - id_value = strtoull(id->value.string.string, &err, 10); + id_value = strtoull(id->string().c_str(), &err, 10); if (err != NULL && *err != '\0') { static bool warned_frac = false; if (!warned_frac) { - fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->value.string.string); + fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->string().c_str()); warned_frac = true; } - } else if (std::to_string(id_value) != id->value.string.string) { + } else if (std::to_string(id_value) != id->string()) { static bool warned = false; if (!warned) { - fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->value.string.string); + fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->string().c_str()); warned = true; } } else { @@ -168,33 +168,29 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom static bool warned_nan = false; if (!warned_nan) { - char *s = json_stringify(id); - fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", s); - free(s); // stringify + fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", json_stringify(id).c_str()); warned_nan = true; } } } } - size_t nprop = 0; - if (properties != NULL && properties->type == JSON_HASH) { - nprop = properties->value.object.length; - } - std::vector> full_keys; std::vector values; - - full_keys.reserve(nprop); - values.reserve(nprop); key_pool key_pool; - for (size_t i = 0; i < nprop; i++) { - if (properties->value.object.keys[i]->type == JSON_STRING) { - serial_val sv = stringify_value(properties->value.object.values[i], sst->fname, sst->line, feature); + if (properties != nullptr && properties->type == JSON_HASH) { + const auto &entries = properties->entries(); + full_keys.reserve(entries.size()); + values.reserve(entries.size()); - full_keys.emplace_back(key_pool.pool(properties->value.object.keys[i]->value.string.string)); - values.push_back(std::move(sv)); + for (const auto &e : entries) { + if (e.key->type == JSON_STRING) { + serial_val sv = stringify_value(e.value.get(), sst->fname, sst->line, feature); + + full_keys.emplace_back(key_pool.pool(e.key->string())); + values.push_back(std::move(sv)); + } } } @@ -220,14 +216,14 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom void check_crs(json_object *j, const char *reading) { json_object *crs = json_hash_get(j, "crs"); - if (crs != NULL) { + if (crs != nullptr) { json_object *properties = json_hash_get(crs, "properties"); - if (properties != NULL) { + if (properties != nullptr) { json_object *name = json_hash_get(properties, "name"); - if (name != NULL && name->type == JSON_STRING) { - if (strcmp(name->value.string.string, projection->alias) != 0) { + if (name != nullptr && name->type == JSON_STRING) { + if (name->string() != projection->alias) { if (!quiet) { - fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->value.string.string, projection->alias); + fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->string().c_str(), projection->alias); fprintf(stderr, "%s: If \"%s\" is not the expected projection, use -s to specify the right one.\n", reading, projection->alias); } } @@ -242,11 +238,13 @@ struct json_serialize_action : json_feature_action { std::string layername; int add_feature(json_object *geometry, bool geometrycollection, json_object *properties, json_object *id, json_object *tippecanoe, json_object *feature) { + // Only json_read results reach this; a detached tree has no parser. + assert(geometry->parser != nullptr); sst->line = geometry->parser->line; if (geometrycollection) { int ret = 1; - for (size_t g = 0; g < geometry->value.array.length; g++) { - ret &= serialize_geojson_feature(sst, geometry->value.array.array[g], properties, id, layer, tippecanoe, feature, layername); + for (size_t g = 0; g < geometry->array().size(); g++) { + ret &= serialize_geojson_feature(sst, geometry->array()[g].get(), properties, id, layer, tippecanoe, feature, layername); } return ret; } else { @@ -259,7 +257,7 @@ struct json_serialize_action : json_feature_action { } }; -void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername) { +void parse_json(struct serialization_state *sst, json_pull_ptr &jp, int layer, std::string layername) { json_serialize_action jsa; jsa.fname = sst->fname; jsa.sst = sst; @@ -296,7 +294,7 @@ ssize_t json_map_read(struct json_pull *jp, char *buffer, size_t n) { return n; } -struct json_pull *json_begin_map(char *map, long long len) { +json_pull_ptr json_begin_map(char *map, long long len) { struct jsonmap *jm = new jsonmap; if (jm == NULL) { perror("Out of memory"); @@ -310,7 +308,11 @@ struct json_pull *json_begin_map(char *map, long long len) { return json_begin(json_map_read, jm); } -void json_end_map(struct json_pull *jp) { +void json_end_map(json_pull_ptr &jp) { + if (jp == nullptr) { + return; + } delete (struct jsonmap *) jp->source; + jp->source = nullptr; json_end(jp); } diff --git a/geojson.hpp b/geojson.hpp index 664ea2e8..3d7b7d26 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -10,21 +10,21 @@ #include "serial.hpp" struct parse_json_args { - json_pull *jp; + json_pull_ptr jp; int layer; std::string *layername; struct serialization_state *sst; - parse_json_args(json_pull *jp1, int layer1, std::string *layername1, struct serialization_state *sst1) - : jp(jp1), layer(layer1), layername(layername1), sst(sst1) { + parse_json_args(json_pull_ptr jp1, int layer1, std::string *layername1, struct serialization_state *sst1) + : jp(std::move(jp1)), layer(layer1), layername(layername1), sst(sst1) { } }; -struct json_pull *json_begin_map(char *map, long long len); -void json_end_map(struct json_pull *jp); +json_pull_ptr json_begin_map(char *map, long long len); +void json_end_map(json_pull_ptr &jp); -void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername); +void parse_json(struct serialization_state *sst, json_pull_ptr &jp, int layer, std::string layername); void *run_parse_json(void *v); #endif diff --git a/jsonpull/jsonpull.c b/jsonpull/jsonpull.c deleted file mode 100644 index 04ca0f29..00000000 --- a/jsonpull/jsonpull.c +++ /dev/null @@ -1,996 +0,0 @@ -#define _GNU_SOURCE // for asprintf() -#include -#include -#include -#include -#include -#include -#include "jsonpull.h" -#include "../milo/milo.h" - -#define BUFFER 10000 - -struct string { - char *buf; - size_t n; - size_t nalloc; -}; - -static void string_init(struct string *s); -static void string_free(struct string *s); - -json_pull *json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source) { - json_pull *j = malloc(sizeof(json_pull)); - if (j == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - j->error = NULL; - j->line = 1; - j->container = NULL; - j->root = NULL; - - j->read = read; - j->source = source; - j->buffer_head = 0; - j->buffer_tail = 0; - - j->buffer = malloc(BUFFER); - if (j->buffer == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - - j->number_buffer = malloc(sizeof(struct string)); - if (j->number_buffer == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - string_init(j->number_buffer); - - return j; -} - -static inline int peek(json_pull *j) { - if (j->buffer_head < j->buffer_tail) { - return (unsigned char) j->buffer[j->buffer_head]; - } else { - j->buffer_head = 0; - j->buffer_tail = j->read(j, j->buffer, BUFFER); - if (j->buffer_head >= j->buffer_tail) { - return EOF; - } - return (unsigned char) j->buffer[j->buffer_head]; - } -} - -static inline int next(json_pull *j) { - if (j->buffer_head < j->buffer_tail) { - return (unsigned char) j->buffer[j->buffer_head++]; - } else { - j->buffer_head = 0; - j->buffer_tail = j->read(j, j->buffer, BUFFER); - if (j->buffer_head >= j->buffer_tail) { - return EOF; - } - return (unsigned char) j->buffer[j->buffer_head++]; - } -} - -static ssize_t read_file(json_pull *j, char *buffer, size_t n) { - return fread(buffer, 1, n, j->source); -} - -json_pull *json_begin_file(FILE *f) { - return json_begin(read_file, f); -} - -static ssize_t read_string(json_pull *j, char *buffer, size_t n) { - const char *cp = j->source; - size_t out = 0; - - while (out < n && cp[out] != '\0') { - buffer[out] = cp[out]; - out++; - } - - j->source = (void *) (cp + out); - return out; -} - -json_pull *json_begin_string(const char *s) { - return json_begin(read_string, (void *) s); -} - -void json_end(json_pull *p) { - string_free(p->number_buffer); - free(p->number_buffer); - - json_free(p->root); - free(p->buffer); - free(p); -} - -static inline int read_wrap(json_pull *j) { - int c = next(j); - - if (c == '\n') { - j->line++; - } - - return c; -} - -#define SIZE_FOR(i, size) ((size_t) ((((i) + 7) & ~7) * size)) - -static json_object *fabricate_object(json_pull *jp, json_object *parent, json_type type) { - json_object *o = malloc(sizeof(struct json_object)); - if (o == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - o->type = type; - o->parent = parent; - o->parser = jp; - - if (type == JSON_ARRAY) { - o->value.array.array = NULL; - o->value.array.length = 0; - } else if (type == JSON_HASH) { - o->value.object.keys = NULL; - o->value.object.values = NULL; - o->value.object.length = 0; - } - - return o; -} - -static json_object *add_object(json_pull *j, json_type type) { - json_object *c = j->container; - json_object *o = fabricate_object(j, c, type); - - if (c != NULL) { - if (c->type == JSON_ARRAY) { - if (c->expect == JSON_ITEM) { - if (SIZE_FOR(c->value.array.length + 1, sizeof(json_object *)) != SIZE_FOR(c->value.array.length, sizeof(json_object *))) { - if (SIZE_FOR(c->value.array.length + 1, sizeof(json_object *)) < SIZE_FOR(c->value.array.length, sizeof(json_object *))) { - fprintf(stderr, "Array size overflow\n"); - exit(EXIT_FAILURE); - } - c->value.array.array = realloc(c->value.array.array, SIZE_FOR(c->value.array.length + 1, sizeof(json_object *))); - if (c->value.array.array == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } - - c->value.array.array[c->value.array.length++] = o; - c->expect = JSON_COMMA; - } else { - j->error = "Expected a comma, not a list item"; - free(o); - return NULL; - } - } else if (c->type == JSON_HASH) { - if (c->expect == JSON_VALUE) { - c->value.object.values[c->value.object.length - 1] = o; - c->expect = JSON_COMMA; - } else if (c->expect == JSON_KEY) { - if (type != JSON_STRING) { - j->error = "Hash key is not a string"; - free(o); - return NULL; - } - - if (SIZE_FOR(c->value.object.length + 1, sizeof(json_object *)) != SIZE_FOR(c->value.object.length, sizeof(json_object *))) { - if (SIZE_FOR(c->value.object.length + 1, sizeof(json_object *)) < SIZE_FOR(c->value.object.length, sizeof(json_object *))) { - fprintf(stderr, "Hash size overflow\n"); - exit(EXIT_FAILURE); - } - c->value.object.keys = realloc(c->value.object.keys, SIZE_FOR(c->value.object.length + 1, sizeof(json_object *))); - c->value.object.values = realloc(c->value.object.values, SIZE_FOR(c->value.object.length + 1, sizeof(json_object *))); - if (c->value.object.keys == NULL || c->value.object.values == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } - - c->value.object.keys[c->value.object.length] = o; - c->value.object.values[c->value.object.length] = NULL; - c->value.object.length++; - c->expect = JSON_COLON; - } else { - j->error = "Expected a comma or colon"; - free(o); - return NULL; - } - } - } else { - if (j->root != NULL) { - json_free(j->root); - } - - j->root = o; - } - - return o; -} - -json_object *json_hash_get(json_object *o, const char *s) { - if (o == NULL || o->type != JSON_HASH) { - return NULL; - } - - size_t i; - for (i = 0; i < o->value.object.length; i++) { - if (o->value.object.keys[i] != NULL && o->value.object.keys[i]->type == JSON_STRING) { - if (strcmp(o->value.object.keys[i]->value.string.string, s) == 0) { - return o->value.object.values[i]; - } - } - } - - return NULL; -} - -static void string_init(struct string *s) { - s->nalloc = 500; - s->buf = malloc(s->nalloc); - if (s->buf == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - s->n = 0; - s->buf[0] = '\0'; -} - -static void string_append(struct string *s, char c) { - if (s->n + 2 >= s->nalloc) { - size_t prev = s->nalloc; - s->nalloc += 500; - if (s->nalloc <= prev) { - fprintf(stderr, "String size overflowed\n"); - exit(EXIT_FAILURE); - } - s->buf = realloc(s->buf, s->nalloc); - if (s->buf == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } - - s->buf[s->n++] = c; - s->buf[s->n] = '\0'; -} - -static void string_append_string(struct string *s, char *add) { - size_t len = strlen(add); - - if (s->n + len + 1 >= s->nalloc) { - size_t prev = s->nalloc; - s->nalloc += 500 + len; - if (s->nalloc <= prev) { - fprintf(stderr, "String size overflowed\n"); - exit(EXIT_FAILURE); - } - s->buf = realloc(s->buf, s->nalloc); - if (s->buf == NULL) { - perror("Out of memory"); - exit(EXIT_FAILURE); - } - } - - for (; *add != '\0'; add++) { - s->buf[s->n++] = *add; - } - - s->buf[s->n] = '\0'; -} - -static void string_free(struct string *s) { - free(s->buf); -} - -json_object *json_read_separators(json_pull *j, json_separator_callback cb, void *state) { - int c; - - // In case there is an error at the top level - if (j->container == NULL) { - if (j->root != NULL) { - json_free(j->root); - } - - j->root = NULL; - } - -again: - c = read_wrap(j); - if (c == EOF) { - if (j->container != NULL) { - j->error = "Reached EOF without all containers being closed"; - } - - return NULL; - } - - switch (c) { - /////////////////////////// Byte order mark - - case 0xEF: { - int c2 = peek(j); - if (c2 == 0xBB) { - c2 = read_wrap(j); - c2 = peek(j); - if (c2 == 0xBF) { - c2 = read_wrap(j); - c = ' '; - goto again; - } - } - j->error = "Corrupt byte-order mark found"; - return NULL; - } - - /////////////////////////// Whitespace - - case ' ': - case '\t': - case '\r': - case '\n': - case 0x1E: - goto again; - - /////////////////////////// Arrays - - case '[': { - json_object *o = add_object(j, JSON_ARRAY); - if (o == NULL) { - return NULL; - } - j->container = o; - j->container->expect = JSON_ITEM; - - if (cb != NULL) { - cb(JSON_ARRAY, j, state); - } - - goto again; - } - - case ']': { - if (j->container == NULL) { - j->error = "Found ] at top level"; - return NULL; - } - - if (j->container->type != JSON_ARRAY) { - j->error = "Found ] not in an array"; - return NULL; - } - - if (j->container->expect != JSON_COMMA) { - if (!(j->container->expect == JSON_ITEM && j->container->value.array.length == 0)) { - j->error = "Found ] without final element"; - return NULL; - } - } - - json_object *ret = j->container; - j->container = ret->parent; - return ret; - } - - /////////////////////////// Hashes - - case '{': { - json_object *o = add_object(j, JSON_HASH); - if (o == NULL) { - return NULL; - } - j->container = o; - j->container->expect = JSON_KEY; - - if (cb != NULL) { - cb(JSON_HASH, j, state); - } - - goto again; - } - - case '}': { - if (j->container == NULL) { - j->error = "Found } at top level"; - return NULL; - } - - if (j->container->type != JSON_HASH) { - j->error = "Found } not in a hash"; - return NULL; - } - - if (j->container->expect != JSON_COMMA) { - if (!(j->container->expect == JSON_KEY && j->container->value.object.length == 0)) { - j->error = "Found } without final element"; - return NULL; - } - } - - json_object *ret = j->container; - j->container = ret->parent; - return ret; - } - - /////////////////////////// Null - - case 'n': { - if (read_wrap(j) != 'u' || read_wrap(j) != 'l' || read_wrap(j) != 'l') { - j->error = "Found misspelling of null"; - return NULL; - } - - return add_object(j, JSON_NULL); - } - - /////////////////////////// NaN - - case 'N': { - if (read_wrap(j) != 'a' || read_wrap(j) != 'N') { - j->error = "Found misspelling of NaN"; - return NULL; - } - - j->error = "JSON does not allow NaN"; - return NULL; - } - - /////////////////////////// Infinity - - case 'I': { - if (read_wrap(j) != 'n' || read_wrap(j) != 'f' || read_wrap(j) != 'i' || - read_wrap(j) != 'n' || read_wrap(j) != 'i' || read_wrap(j) != 't' || - read_wrap(j) != 'y') { - j->error = "Found misspelling of Infinity"; - return NULL; - } - - j->error = "JSON does not allow Infinity"; - return NULL; - } - - /////////////////////////// True - - case 't': { - if (read_wrap(j) != 'r' || read_wrap(j) != 'u' || read_wrap(j) != 'e') { - j->error = "Found misspelling of true"; - return NULL; - } - - return add_object(j, JSON_TRUE); - } - - /////////////////////////// False - - case 'f': { - if (read_wrap(j) != 'a' || read_wrap(j) != 'l' || read_wrap(j) != 's' || read_wrap(j) != 'e') { - j->error = "Found misspelling of false"; - return NULL; - } - - return add_object(j, JSON_FALSE); - } - - /////////////////////////// Comma - - case ',': { - if (j->container != NULL) { - if (j->container->expect != JSON_COMMA) { - j->error = "Found unexpected comma"; - return NULL; - } - - if (j->container->type == JSON_HASH) { - j->container->expect = JSON_KEY; - } else { - j->container->expect = JSON_ITEM; - } - } - - if (cb != NULL) { - cb(JSON_COMMA, j, state); - } - - goto again; - } - - /////////////////////////// Colon - - case ':': { - if (j->container == NULL) { - j->error = "Found colon at top level"; - return NULL; - } - - if (j->container->expect != JSON_COLON) { - j->error = "Found unexpected colon"; - return NULL; - } - - j->container->expect = JSON_VALUE; - - if (cb != NULL) { - cb(JSON_COLON, j, state); - } - - goto again; - } - - /////////////////////////// Numbers - - case '-': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - j->number_buffer->n = 0; - int decimal = 0; - - if (c == '-') { - string_append(j->number_buffer, c); - c = read_wrap(j); - } - - if (c == '0') { - string_append(j->number_buffer, c); - } else if (c >= '1' && c <= '9') { - string_append(j->number_buffer, c); - c = peek(j); - - while (c >= '0' && c <= '9') { - string_append(j->number_buffer, read_wrap(j)); - c = peek(j); - } - } - - if (peek(j) == '.') { - string_append(j->number_buffer, read_wrap(j)); - decimal = 1; - - c = peek(j); - if (c < '0' || c > '9') { - j->error = "Decimal point without digits"; - string_free(j->number_buffer); - return NULL; - } - while (c >= '0' && c <= '9') { - string_append(j->number_buffer, read_wrap(j)); - c = peek(j); - } - } - - c = peek(j); - if (c == 'e' || c == 'E') { - string_append(j->number_buffer, read_wrap(j)); - decimal = 1; - - c = peek(j); - if (c == '+' || c == '-') { - string_append(j->number_buffer, read_wrap(j)); - } - - c = peek(j); - if (c < '0' || c > '9') { - j->error = "Exponent without digits"; - string_free(j->number_buffer); - return NULL; - } - while (c >= '0' && c <= '9') { - string_append(j->number_buffer, read_wrap(j)); - c = peek(j); - } - } - - json_object *n = add_object(j, JSON_NUMBER); - if (n != NULL) { - n->value.number.number = atof(j->number_buffer->buf); - n->value.number.large_signed = 0; - n->value.number.large_unsigned = 0; - -#define MAX_SAFE_INTEGER 9007199254740991.0 -#define MIN_SAFE_INTEGER -9007199254740991.0 - - if (!decimal && n->value.number.number > MAX_SAFE_INTEGER) { - errno = 0; - char *err = NULL; - unsigned long long ull = strtoull(j->number_buffer->buf, &err, 10); - if (errno == 0 && (err == NULL || *err == '\0')) { - n->value.number.large_unsigned = ull; - } - } - if (!decimal && n->value.number.number < MIN_SAFE_INTEGER) { - errno = 0; - char *err = NULL; - long long ll = strtoll(j->number_buffer->buf, &err, 10); - if (errno == 0 && (err == NULL || *err == '\0')) { - n->value.number.large_signed = ll; - } - } - } - return n; - } - - /////////////////////////// Strings - - case '"': { - struct string val; - string_init(&val); - - int surrogate = -1; - while ((c = read_wrap(j)) != EOF) { - if (c == '"') { - if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = -1; - } - - break; - } else if (c == '\\') { - c = read_wrap(j); - - if (c == 'u') { - char hex[5] = "aaaa"; - int i; - for (i = 0; i < 4; i++) { - hex[i] = read_wrap(j); - if (hex[i] < '0' || (hex[i] > '9' && hex[i] < 'A') || (hex[i] > 'F' && hex[i] < 'a') || hex[i] > 'f') { - j->error = "Invalid \\u hex character"; - string_free(&val); - return NULL; - } - } - - unsigned long ch = strtoul(hex, NULL, 16); - if (ch >= 0xd800 && ch <= 0xdbff) { - if (surrogate < 0) { - surrogate = ch; - } else { - // Impossible surrogate, so output the first half, - // keep what might be a legitimate new first half. - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = ch; - } - continue; - } else if (ch >= 0xdc00 && c <= 0xdfff) { - if (surrogate >= 0) { - long c1 = surrogate - 0xd800; - long c2 = ch - 0xdc00; - ch = ((c1 << 10) | c2) + 0x010000; - surrogate = -1; - } - } - - if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = -1; - } - - if (ch <= 0x7F) { - string_append(&val, ch); - } else if (ch <= 0x7FF) { - string_append(&val, 0xC0 | (ch >> 6)); - string_append(&val, 0x80 | (ch & 0x3F)); - } else if (ch < 0xFFFF) { - string_append(&val, 0xE0 | (ch >> 12)); - string_append(&val, 0x80 | ((ch >> 6) & 0x3F)); - string_append(&val, 0x80 | (ch & 0x3F)); - } else { - string_append(&val, 0xF0 | (ch >> 18)); - string_append(&val, 0x80 | ((ch >> 12) & 0x3F)); - string_append(&val, 0x80 | ((ch >> 6) & 0x3F)); - string_append(&val, 0x80 | (ch & 0x3F)); - } - } else { - if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = -1; - } - - if (c == '"') { - string_append(&val, '"'); - } else if (c == '\\') { - string_append(&val, '\\'); - } else if (c == '/') { - string_append(&val, '/'); - } else if (c == 'b') { - string_append(&val, '\b'); - } else if (c == 'f') { - string_append(&val, '\f'); - } else if (c == 'n') { - string_append(&val, '\n'); - } else if (c == 'r') { - string_append(&val, '\r'); - } else if (c == 't') { - string_append(&val, '\t'); - } else { - j->error = "Found backslash followed by unknown character"; - string_free(&val); - return NULL; - } - } - } else if (c < ' ') { - j->error = "Found control character in string"; - string_free(&val); - return NULL; - } else { - if (surrogate >= 0) { - string_append(&val, 0xE0 | (surrogate >> 12)); - string_append(&val, 0x80 | ((surrogate >> 6) & 0x3F)); - string_append(&val, 0x80 | (surrogate & 0x3F)); - surrogate = -1; - } - - string_append(&val, c); - } - } - if (c == EOF) { - j->error = "String without closing quote mark"; - string_free(&val); - return NULL; - } - - json_object *s = add_object(j, JSON_STRING); - if (s != NULL) { - s->value.string.string = val.buf; - s->value.string.refcon = NULL; - } else { - string_free(&val); - } - return s; - } - } - - j->error = "Found unexpected character"; - return NULL; -} - -json_object *json_read(json_pull *j) { - return json_read_separators(j, NULL, NULL); -} - -json_object *json_read_tree(json_pull *p) { - json_object *j; - - while ((j = json_read(p)) != NULL) { - if (j->parent == NULL) { - return j; - } - } - - return NULL; -} - -void json_free(json_object *o) { - size_t i; - - if (o == NULL) { - return; - } - - // Free any data linked from here - - if (o->type == JSON_ARRAY) { - json_object **a = o->value.array.array; - size_t n = o->value.array.length; - - o->value.array.array = NULL; - o->value.array.length = 0; - - for (i = 0; i < n; i++) { - json_free(a[i]); - } - - free(a); - } else if (o->type == JSON_HASH) { - json_object **k = o->value.object.keys; - json_object **v = o->value.object.values; - size_t n = o->value.object.length; - - o->value.object.keys = NULL; - o->value.object.values = NULL; - o->value.object.length = 0; - - for (i = 0; i < n; i++) { - json_free(k[i]); - json_free(v[i]); - } - - free(k); - free(v); - } else if (o->type == JSON_STRING) { - free(o->value.string.string); - } else if (o->type == JSON_NUMBER) { - ; - } - - json_disconnect(o); - - free(o); -} - -static void json_disconnect_parser(json_object *o) { - if (o->type == JSON_HASH) { - size_t i; - for (i = 0; i < o->value.object.length; i++) { - json_disconnect_parser(o->value.object.keys[i]); - json_disconnect_parser(o->value.object.values[i]); - } - } else if (o->type == JSON_ARRAY) { - size_t i; - for (i = 0; i < o->value.array.length; i++) { - json_disconnect_parser(o->value.array.array[i]); - } - } - - o->parser = NULL; -} - -void json_disconnect(json_object *o) { - // Expunge references to this as an array element - // or a hash key or value. - - if (o->parent != NULL) { - if (o->parent->type == JSON_ARRAY) { - size_t i; - - for (i = 0; i < o->parent->value.array.length; i++) { - if (o->parent->value.array.array[i] == o) { - break; - } - } - - if (i < o->parent->value.array.length) { - memmove(o->parent->value.array.array + i, o->parent->value.array.array + i + 1, o->parent->value.array.length - i - 1); - o->parent->value.array.length--; - } - } - - if (o->parent->type == JSON_HASH) { - size_t i; - - for (i = 0; i < o->parent->value.object.length; i++) { - if (o->parent->value.object.keys[i] == o) { - o->parent->value.object.keys[i] = fabricate_object(o->parser, o->parent, JSON_NULL); - break; - } - if (o->parent->value.object.values[i] == o) { - o->parent->value.object.values[i] = fabricate_object(o->parser, o->parent, JSON_NULL); - break; - } - } - - if (i < o->parent->value.object.length) { - if (o->parent->value.object.keys[i] != NULL && o->parent->value.object.keys[i]->type == JSON_NULL) { - if (o->parent->value.object.values[i] != NULL && o->parent->value.object.values[i]->type == JSON_NULL) { - free(o->parent->value.object.keys[i]); - free(o->parent->value.object.values[i]); - - memmove(o->parent->value.object.keys + i, o->parent->value.object.keys + i + 1, o->parent->value.object.length - i - 1); - memmove(o->parent->value.object.values + i, o->parent->value.object.values + i + 1, o->parent->value.object.length - i - 1); - o->parent->value.object.length--; - } - } - } - } - } - - if (o->parser != NULL && o->parser->root == o) { - o->parser->root = NULL; - } - - json_disconnect_parser(o); - o->parent = NULL; -} - -static void json_print_one(struct string *val, json_object *o) { - if (o == NULL) { - string_append_string(val, "..."); - } else if (o->type == JSON_STRING) { - string_append(val, '\"'); - - char *cp; - for (cp = o->value.string.string; *cp != '\0'; cp++) { - if (*cp == '\\' || *cp == '"') { - string_append(val, '\\'); - string_append(val, *cp); - } else if (*cp >= 0 && *cp < ' ') { - char *s; - if (asprintf(&s, "\\u%04x", *cp) >= 0) { - string_append_string(val, s); - free(s); - } - } else { - string_append(val, *cp); - } - } - - string_append(val, '\"'); - } else if (o->type == JSON_NUMBER) { - if (o->value.number.large_signed != 0) { - char s[65]; - sprintf(s, "%lld", o->value.number.large_signed); - string_append_string(val, s); - } else if (o->value.number.large_unsigned != 0) { - char s[65]; - sprintf(s, "%llu", o->value.number.large_unsigned); - string_append_string(val, s); - } else { - char *s = dtoa_milo(o->value.number.number); - string_append_string(val, s); - free(s); - } - } else if (o->type == JSON_NULL) { - string_append_string(val, "null"); - } else if (o->type == JSON_TRUE) { - string_append_string(val, "true"); - } else if (o->type == JSON_FALSE) { - string_append_string(val, "false"); - } else if (o->type == JSON_HASH) { - string_append(val, '}'); - } else if (o->type == JSON_ARRAY) { - string_append(val, ']'); - } -} - -static void json_print(struct string *val, json_object *o) { - if (o == NULL) { - // Hash value in incompletely read hash - string_append_string(val, "..."); - } else if (o->type == JSON_HASH) { - string_append(val, '{'); - - size_t i; - for (i = 0; i < o->value.object.length; i++) { - json_print(val, o->value.object.keys[i]); - string_append(val, ':'); - json_print(val, o->value.object.values[i]); - if (i + 1 < o->value.object.length) { - string_append(val, ','); - } - } - string_append(val, '}'); - } else if (o->type == JSON_ARRAY) { - string_append(val, '['); - size_t i; - for (i = 0; i < o->value.array.length; i++) { - json_print(val, o->value.array.array[i]); - if (i + 1 < o->value.array.length) { - string_append(val, ','); - } - } - string_append(val, ']'); - } else { - json_print_one(val, o); - } -} - -char *json_stringify(json_object *o) { - struct string val; - string_init(&val); - json_print(&val, o); - - return val.buf; -} diff --git a/jsonpull/jsonpull.cpp b/jsonpull/jsonpull.cpp new file mode 100644 index 00000000..c5aa5027 --- /dev/null +++ b/jsonpull/jsonpull.cpp @@ -0,0 +1,857 @@ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE // for asprintf() +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "jsonpull.h" +#include "../milo/milo.h" + +#define BUFFER 10000 + +json_pull_ptr json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source) { + auto j = std::make_shared(); + j->read = read; + j->source = source; + j->buffer.resize(BUFFER); + return j; +} + +static inline int peek(json_pull *j) { + if (j->buffer_head < j->buffer_tail) { + return (unsigned char) j->buffer[j->buffer_head]; + } else { + j->buffer_head = 0; + j->buffer_tail = j->read(j, j->buffer.data(), BUFFER); + if (j->buffer_head >= j->buffer_tail) { + return EOF; + } + return (unsigned char) j->buffer[j->buffer_head]; + } +} + +static inline int next(json_pull *j) { + if (j->buffer_head < j->buffer_tail) { + return (unsigned char) j->buffer[j->buffer_head++]; + } else { + j->buffer_head = 0; + j->buffer_tail = j->read(j, j->buffer.data(), BUFFER); + if (j->buffer_head >= j->buffer_tail) { + return EOF; + } + return (unsigned char) j->buffer[j->buffer_head++]; + } +} + +static ssize_t read_file(json_pull *j, char *buffer, size_t n) { + return fread(buffer, 1, n, (FILE *) j->source); +} + +json_pull_ptr json_begin_file(FILE *f) { + return json_begin(read_file, f); +} + +static ssize_t read_string(json_pull *j, char *buffer, size_t n) { + const char *cp = (const char *) j->source; + size_t out = 0; + + while (out < n && cp[out] != '\0') { + buffer[out] = cp[out]; + out++; + } + + j->source = (void *) (cp + out); + return out; +} + +json_pull_ptr json_begin_string(const char *s) { + return json_begin(read_string, (void *) s); +} + +void json_end(json_pull_ptr &p) { + p.reset(); +} + +static inline int read_wrap(json_pull *j) { + int c = next(j); + + if (c == '\n') { + j->line++; + } + + return c; +} + +// JSON_TRUE / JSON_FALSE / JSON_NULL and the parse-token types are bare +// json_objects; the value-bearing types each get their own subclass. +static json_object_ptr make_object(json_type type, json_object *parent, json_pull *jp) { + switch (type) { + case JSON_NUMBER: + return json_object_ptr(new json_number(parent, jp)); + case JSON_STRING: + return json_object_ptr(new json_string(parent, jp)); + case JSON_ARRAY: + return json_object_ptr(new json_array(parent, jp)); + case JSON_HASH: + return json_object_ptr(new json_hash(parent, jp)); + default: + return json_object_ptr(new json_object(type, parent, jp)); + } +} + +static inline json_pull::parse_frame *current_frame(json_pull *j) { + return j->container_stack.empty() ? nullptr : &j->container_stack.back(); +} + +// Install a new node of `type` in the current container, or as the parser's +// root if the stack is empty. Returns it borrowed, or nullptr after setting +// j->error. +static json_object *add_object(json_pull *j, json_type type) { + json_pull::parse_frame *f = current_frame(j); + json_object *c = f ? f->container : nullptr; + json_object_ptr o = make_object(type, c, j); + json_object *raw = o.get(); + + if (f != nullptr) { + if (c->type == JSON_ARRAY) { + if (f->expect == JSON_ITEM) { + c->array().push_back(std::move(o)); + f->expect = JSON_COMMA; + } else { + j->error = "Expected a comma, not a list item"; + return nullptr; + } + } else if (c->type == JSON_HASH) { + if (f->expect == JSON_VALUE) { + // A colon is the only thing that sets JSON_VALUE, and it + // requires a key already pushed. + assert(!c->entries().empty()); + c->entries().back().value = std::move(o); + f->expect = JSON_COMMA; + } else if (f->expect == JSON_KEY) { + if (type != JSON_STRING) { + j->error = "Hash key is not a string"; + return nullptr; + } + + c->entries().push_back({std::move(o), nullptr}); + f->expect = JSON_COLON; + } else { + j->error = "Expected a comma or colon"; + return nullptr; + } + } + } else { + // Replacing the parser's root destroys the previous top-level + // value (if no one called json_disconnect / json_read_tree to + // take ownership of it). + j->root = std::move(o); + } + + return raw; +} + +json_object *json_hash_get(json_object *o, const char *s) { + if (o == nullptr || o->type != JSON_HASH) { + return nullptr; + } + + for (const auto &e : o->entries()) { + if (e.key != nullptr && e.key->type == JSON_STRING && e.key->string() == s) { + return e.value.get(); + } + } + + return nullptr; +} + +json_object *json_hash_get(const json_object_ptr &o, const char *s) { + return json_hash_get(o.get(), s); +} + +json_object *json_read_separators(json_pull_ptr &jp, json_separator_callback cb, void *state) { + int c; + json_pull *j = jp.get(); + + // In case there is an error at the top level + if (j->container_stack.empty()) { + j->root.reset(); + } + +again: + c = read_wrap(j); + if (c == EOF) { + if (!j->container_stack.empty()) { + j->error = "Reached EOF without all containers being closed"; + } + + return nullptr; + } + + switch (c) { + /////////////////////////// Byte order mark + + case 0xEF: { + int c2 = peek(j); + if (c2 == 0xBB) { + c2 = read_wrap(j); + c2 = peek(j); + if (c2 == 0xBF) { + c2 = read_wrap(j); + c = ' '; + goto again; + } + } + j->error = "Corrupt byte-order mark found"; + return nullptr; + } + + /////////////////////////// Whitespace + + case ' ': + case '\t': + case '\r': + case '\n': + case 0x1E: + goto again; + + /////////////////////////// Arrays + + case '[': { + json_object *o = add_object(j, JSON_ARRAY); + if (o == nullptr) { + return nullptr; + } + j->container_stack.push_back({o, JSON_ITEM}); + + if (cb != nullptr) { + cb(JSON_ARRAY, j, state); + } + + goto again; + } + + case ']': { + json_pull::parse_frame *f = current_frame(j); + if (f == nullptr) { + j->error = "Found ] at top level"; + return nullptr; + } + + json_object *cc = f->container; + if (cc->type != JSON_ARRAY) { + j->error = "Found ] not in an array"; + return nullptr; + } + + if (f->expect != JSON_COMMA) { + if (!(f->expect == JSON_ITEM && cc->array().size() == 0)) { + j->error = "Found ] without final element"; + return nullptr; + } + } + + j->container_stack.pop_back(); + return cc; + } + + /////////////////////////// Hashes + + case '{': { + json_object *o = add_object(j, JSON_HASH); + if (o == nullptr) { + return nullptr; + } + j->container_stack.push_back({o, JSON_KEY}); + + if (cb != nullptr) { + cb(JSON_HASH, j, state); + } + + goto again; + } + + case '}': { + json_pull::parse_frame *f = current_frame(j); + if (f == nullptr) { + j->error = "Found } at top level"; + return nullptr; + } + + json_object *cc = f->container; + if (cc->type != JSON_HASH) { + j->error = "Found } not in a hash"; + return nullptr; + } + + if (f->expect != JSON_COMMA) { + if (!(f->expect == JSON_KEY && cc->entries().size() == 0)) { + j->error = "Found } without final element"; + return nullptr; + } + } + + j->container_stack.pop_back(); + return cc; + } + + /////////////////////////// Null + + case 'n': { + if (read_wrap(j) != 'u' || read_wrap(j) != 'l' || read_wrap(j) != 'l') { + j->error = "Found misspelling of null"; + return nullptr; + } + + return add_object(j, JSON_NULL); + } + + /////////////////////////// NaN + + case 'N': { + if (read_wrap(j) != 'a' || read_wrap(j) != 'N') { + j->error = "Found misspelling of NaN"; + return nullptr; + } + + j->error = "JSON does not allow NaN"; + return nullptr; + } + + /////////////////////////// Infinity + + case 'I': { + if (read_wrap(j) != 'n' || read_wrap(j) != 'f' || read_wrap(j) != 'i' || + read_wrap(j) != 'n' || read_wrap(j) != 'i' || read_wrap(j) != 't' || + read_wrap(j) != 'y') { + j->error = "Found misspelling of Infinity"; + return nullptr; + } + + j->error = "JSON does not allow Infinity"; + return nullptr; + } + + /////////////////////////// True + + case 't': { + if (read_wrap(j) != 'r' || read_wrap(j) != 'u' || read_wrap(j) != 'e') { + j->error = "Found misspelling of true"; + return nullptr; + } + + return add_object(j, JSON_TRUE); + } + + /////////////////////////// False + + case 'f': { + if (read_wrap(j) != 'a' || read_wrap(j) != 'l' || read_wrap(j) != 's' || read_wrap(j) != 'e') { + j->error = "Found misspelling of false"; + return nullptr; + } + + return add_object(j, JSON_FALSE); + } + + /////////////////////////// Comma + + case ',': { + json_pull::parse_frame *f = current_frame(j); + if (f != nullptr) { + if (f->expect != JSON_COMMA) { + j->error = "Found unexpected comma"; + return nullptr; + } + + if (f->container->type == JSON_HASH) { + f->expect = JSON_KEY; + } else { + f->expect = JSON_ITEM; + } + } + + if (cb != nullptr) { + cb(JSON_COMMA, j, state); + } + + goto again; + } + + /////////////////////////// Colon + + case ':': { + json_pull::parse_frame *f = current_frame(j); + if (f == nullptr) { + j->error = "Found colon at top level"; + return nullptr; + } + + if (f->expect != JSON_COLON) { + j->error = "Found unexpected colon"; + return nullptr; + } + + f->expect = JSON_VALUE; + + if (cb != nullptr) { + cb(JSON_COLON, j, state); + } + + goto again; + } + + /////////////////////////// Numbers + + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + j->number_buffer.clear(); + int decimal = 0; + + if (c == '-') { + j->number_buffer.push_back(c); + c = read_wrap(j); + } + + if (c == '0') { + j->number_buffer.push_back(c); + } else if (c >= '1' && c <= '9') { + j->number_buffer.push_back(c); + c = peek(j); + + while (c >= '0' && c <= '9') { + j->number_buffer.push_back(read_wrap(j)); + c = peek(j); + } + } + + if (peek(j) == '.') { + j->number_buffer.push_back(read_wrap(j)); + decimal = 1; + + c = peek(j); + if (c < '0' || c > '9') { + j->error = "Decimal point without digits"; + return nullptr; + } + while (c >= '0' && c <= '9') { + j->number_buffer.push_back(read_wrap(j)); + c = peek(j); + } + } + + c = peek(j); + if (c == 'e' || c == 'E') { + j->number_buffer.push_back(read_wrap(j)); + decimal = 1; + + c = peek(j); + if (c == '+' || c == '-') { + j->number_buffer.push_back(read_wrap(j)); + } + + c = peek(j); + if (c < '0' || c > '9') { + j->error = "Exponent without digits"; + return nullptr; + } + while (c >= '0' && c <= '9') { + j->number_buffer.push_back(read_wrap(j)); + c = peek(j); + } + } + + json_object *n = add_object(j, JSON_NUMBER); + if (n != nullptr) { + double d = atof(j->number_buffer.c_str()); + n->set_number(d); + +#define MAX_SAFE_INTEGER 9007199254740991.0 +#define MIN_SAFE_INTEGER -9007199254740991.0 + + if (!decimal && d > MAX_SAFE_INTEGER) { + errno = 0; + char *err = nullptr; + unsigned long long ull = strtoull(j->number_buffer.c_str(), &err, 10); + if (errno == 0 && (err == nullptr || *err == '\0')) { + n->set_large_unsigned(ull); + } + } + if (!decimal && d < MIN_SAFE_INTEGER) { + errno = 0; + char *err = nullptr; + long long ll = strtoll(j->number_buffer.c_str(), &err, 10); + if (errno == 0 && (err == nullptr || *err == '\0')) { + n->set_large_signed(ll); + } + } + } + return n; + } + + /////////////////////////// Strings + + case '"': { + // Reused across tokens; see json_pull::string_buffer. + std::string &val = j->string_buffer; + val.clear(); + + int surrogate = -1; + while ((c = read_wrap(j)) != EOF) { + if (c == '"') { + if (surrogate >= 0) { + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = -1; + } + + break; + } else if (c == '\\') { + c = read_wrap(j); + + if (c == 'u') { + char hex[5] = "aaaa"; + int i; + for (i = 0; i < 4; i++) { + hex[i] = read_wrap(j); + if (hex[i] < '0' || (hex[i] > '9' && hex[i] < 'A') || (hex[i] > 'F' && hex[i] < 'a') || hex[i] > 'f') { + j->error = "Invalid \\u hex character"; + return nullptr; + } + } + + unsigned long ch = strtoul(hex, nullptr, 16); + if (ch >= 0xd800 && ch <= 0xdbff) { + if (surrogate < 0) { + surrogate = ch; + } else { + // Impossible surrogate, so output the first half, + // keep what might be a legitimate new first half. + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = ch; + } + continue; + } else if (ch >= 0xdc00 && ch <= 0xdfff) { + if (surrogate >= 0) { + long c1 = surrogate - 0xd800; + long c2 = ch - 0xdc00; + ch = ((c1 << 10) | c2) + 0x010000; + surrogate = -1; + } + } + + if (surrogate >= 0) { + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = -1; + } + + if (ch <= 0x7F) { + val.push_back(ch); + } else if (ch <= 0x7FF) { + val.push_back(0xC0 | (ch >> 6)); + val.push_back(0x80 | (ch & 0x3F)); + } else if (ch <= 0xFFFF) { + val.push_back(0xE0 | (ch >> 12)); + val.push_back(0x80 | ((ch >> 6) & 0x3F)); + val.push_back(0x80 | (ch & 0x3F)); + } else { + // Only reachable for a code point assembled from a + // surrogate pair above, since `ch` on its own comes + // from four hex digits and so cannot exceed 0xFFFF. + val.push_back(0xF0 | (ch >> 18)); + val.push_back(0x80 | ((ch >> 12) & 0x3F)); + val.push_back(0x80 | ((ch >> 6) & 0x3F)); + val.push_back(0x80 | (ch & 0x3F)); + } + } else { + if (surrogate >= 0) { + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = -1; + } + + if (c == '"') { + val.push_back('"'); + } else if (c == '\\') { + val.push_back('\\'); + } else if (c == '/') { + val.push_back('/'); + } else if (c == 'b') { + val.push_back('\b'); + } else if (c == 'f') { + val.push_back('\f'); + } else if (c == 'n') { + val.push_back('\n'); + } else if (c == 'r') { + val.push_back('\r'); + } else if (c == 't') { + val.push_back('\t'); + } else { + j->error = "Found backslash followed by unknown character"; + return nullptr; + } + } + } else if (c < ' ') { + j->error = "Found control character in string"; + return nullptr; + } else { + if (surrogate >= 0) { + val.push_back(0xE0 | (surrogate >> 12)); + val.push_back(0x80 | ((surrogate >> 6) & 0x3F)); + val.push_back(0x80 | (surrogate & 0x3F)); + surrogate = -1; + } + + val.push_back(c); + } + } + if (c == EOF) { + j->error = "String without closing quote mark"; + return nullptr; + } + + json_object *s = add_object(j, JSON_STRING); + if (s != nullptr) { + // Copy, not move, so the buffer keeps its capacity. + s->string() = val; + } + return s; + } + } + + j->error = "Found unexpected character"; + return nullptr; +} + +json_object *json_read(json_pull_ptr &j) { + return json_read_separators(j, nullptr, nullptr); +} + +static void detach_subtree(json_object *o); + +json_object_ptr json_read_tree(json_pull_ptr &p) { + json_object *j; + + while ((j = json_read(p)) != nullptr) { + if (j->parent == nullptr) { + json_object_ptr tree = std::move(p->root); + detach_subtree(tree.get()); + return tree; + } + } + + return nullptr; +} + +// Move the owning json_object_ptr out of whatever vector slot or hash entry +// holds `o`. Empty if nothing tracked owns it -- already detached, or borrowed +// from elsewhere. +// +// Detaching one half of a hash entry would disturb the surrounding key/value +// pairing, so the extracted half is replaced by a JSON_NULL placeholder and the +// entry is erased only once both halves are gone. +static json_object_ptr take_from_owner(json_object *o) { + if (o == nullptr) { + return nullptr; + } + + json_object *parent = o->parent; + if (parent == nullptr) { + // Top-level value: the parser owns it via root, unless the + // caller already moved it out. + json_pull *parser = o->parser; + if (parser != nullptr && parser->root.get() == o) { + return std::move(parser->root); + } + return nullptr; + } + + if (parent->type == JSON_ARRAY) { + auto &arr = parent->array(); + for (size_t i = 0; i < arr.size(); i++) { + if (arr[i].get() == o) { + json_object_ptr taken = std::move(arr[i]); + arr.erase(arr.begin() + i); + return taken; + } + } + } else if (parent->type == JSON_HASH) { + auto &entries = parent->entries(); + for (size_t i = 0; i < entries.size(); i++) { + auto &e = entries[i]; + if (e.key.get() == o) { + json_object_ptr taken = std::move(e.key); + e.key = make_object(JSON_NULL, parent, parent->parser); + if (e.value != nullptr && e.value->type == JSON_NULL && e.key->type == JSON_NULL) { + entries.erase(entries.begin() + i); + } + return taken; + } + if (e.value.get() == o) { + json_object_ptr taken = std::move(e.value); + e.value = make_object(JSON_NULL, parent, parent->parser); + if (e.key != nullptr && e.key->type == JSON_NULL && e.value->type == JSON_NULL) { + entries.erase(entries.begin() + i); + } + return taken; + } + } + } + + return nullptr; +} + +// Splice `o` out of its owner and destroy it; `o` dangles afterwards. No need +// to clear back-pointers, since nothing will observe them again. +void json_free(json_object *o) { + (void) take_from_owner(o); +} + +// See Ownership model in jsonpull.h for why only `parser` is cleared here. +static void clear_parser_pointers(json_object *o) { + if (o == nullptr) { + return; + } + + if (o->type == JSON_HASH) { + for (const auto &e : o->entries()) { + clear_parser_pointers(e.key.get()); + clear_parser_pointers(e.value.get()); + } + } else if (o->type == JSON_ARRAY) { + const auto &arr = o->array(); + for (size_t i = 0; i < arr.size(); i++) { + clear_parser_pointers(arr[i].get()); + } + } + + o->parser = nullptr; +} + +// The root's `parent` pointed out of the subtree, at a node the parser still +// owns; leaving it set would make a later json_free look for this node in a +// container that no longer holds it. +static void detach_subtree(json_object *o) { + if (o == nullptr) { + return; + } + + clear_parser_pointers(o); + o->parent = nullptr; +} + +json_object_ptr json_disconnect(json_object *o) { + json_object_ptr taken = take_from_owner(o); + if (taken != nullptr) { + detach_subtree(taken.get()); + } + return taken; +} + +static void json_print_one(std::string &val, const json_object *o) { + if (o == nullptr) { + val.append("..."); + } else if (o->type == JSON_STRING) { + val.push_back('\"'); + + // Range, not c_str(): the value may contain an embedded NUL, which the + // control-character branch below escapes like any other. + for (char c : o->string()) { + if (c == '\\' || c == '"') { + val.push_back('\\'); + val.push_back(c); + } else if (c >= 0 && c < ' ') { + char *s; + if (asprintf(&s, "\\u%04x", c) >= 0) { + val.append(s); + free(s); + } + } else { + val.push_back(c); + } + } + + val.push_back('\"'); + } else if (o->type == JSON_NUMBER) { + if (o->large_signed() != 0) { + char s[65]; + snprintf(s, sizeof(s), "%lld", o->large_signed()); + val.append(s); + } else if (o->large_unsigned() != 0) { + char s[65]; + snprintf(s, sizeof(s), "%llu", o->large_unsigned()); + val.append(s); + } else { + char *s = dtoa_milo(o->number()); + val.append(s); + free(s); + } + } else if (o->type == JSON_NULL) { + val.append("null"); + } else if (o->type == JSON_TRUE) { + val.append("true"); + } else if (o->type == JSON_FALSE) { + val.append("false"); + } + // JSON_HASH and JSON_ARRAY never reach here: json_print handles both + // itself and only delegates to json_print_one for the scalar types. +} + +static void json_print(std::string &val, const json_object *o) { + if (o == nullptr) { + // Hash value in incompletely read hash + val.append("..."); + } else if (o->type == JSON_HASH) { + val.push_back('{'); + + const auto &entries = o->entries(); + for (size_t i = 0; i < entries.size(); i++) { + json_print(val, entries[i].key.get()); + val.push_back(':'); + json_print(val, entries[i].value.get()); + if (i + 1 < entries.size()) { + val.push_back(','); + } + } + val.push_back('}'); + } else if (o->type == JSON_ARRAY) { + val.push_back('['); + const auto &arr = o->array(); + for (size_t i = 0; i < arr.size(); i++) { + json_print(val, arr[i].get()); + if (i + 1 < arr.size()) { + val.push_back(','); + } + } + val.push_back(']'); + } else { + json_print_one(val, o); + } +} + +std::string json_stringify(const json_object *o) { + std::string val; + json_print(val, o); + return val; +} diff --git a/jsonpull/jsonpull.h b/jsonpull/jsonpull.h index b19e0e8e..5bb7ca63 100644 --- a/jsonpull/jsonpull.h +++ b/jsonpull/jsonpull.h @@ -1,9 +1,12 @@ #ifndef JSONPULL_H #define JSONPULL_H -#ifdef __cplusplus -extern "C" { -#endif +#include +#include +#include +#include +#include +#include typedef enum json_type { // These types can be returned by json_read() @@ -25,74 +28,298 @@ typedef enum json_type { JSON_VALUE, } json_type; -typedef struct json_object { - struct json_object *parent; - struct json_pull *parser; +struct json_object; +struct json_pull; - union { - struct { - double number; - unsigned long long large_unsigned; - long long large_signed; - } number; +// Ownership model +// +// Every node has exactly one owner: its parent (a json_object_ptr in the +// parent's vector or hash entry), the parser (jp->root) for a top-level value, +// or the caller once json_read_tree / json_disconnect hands the tree over. +// json_read and json_hash_get return borrowed pointers, valid while the owning +// container is intact. +// +// `parent` and `parser` are non-owning, so they cannot form a cycle. Detaching +// clears every `parser` in the subtree, since the json_pull may die first, and +// clears `parent` only on the detached root, which pointed out of the subtree. +// Interior `parent` links stay, so a detached tree is still walkable upwards +// and json_free / json_disconnect still work inside it -- both find a node's +// owner through `parent`. +// +// json_object has no vptr; json_object_deleter switches on `type` and +// static_casts so the right subclass destructor runs. Payloads live in those +// subclasses rather than one wide struct, and the accessors assert on `type` +// before downcasting. +// +// json_pull_ptr is a shared_ptr: a parser is created and freed once. - struct { - char *string; - void *refcon; // reference constant for caller's use - } string; +// Stateless, so json_object_ptr stays one pointer wide. See Ownership model. +struct json_object_deleter { + void operator()(json_object *p) const noexcept; +}; - struct { - struct json_object **array; - size_t length; - } array; +typedef std::unique_ptr json_object_ptr; +typedef std::shared_ptr json_pull_ptr; - struct { - struct json_object **keys; - struct json_object **values; - size_t length; - } object; - } value; +// One key/value pair in a JSON_HASH, held in source order. +struct json_entry { + json_object_ptr key; + json_object_ptr value; +}; + +struct json_object { + json_object *parent = nullptr; + json_pull *parser = nullptr; json_type type; - int expect; -} json_object; -typedef struct json_pull { - char *error; - int line; + json_object(json_type t) + : type(t) { + } + json_object(json_type t, json_object *p, json_pull *pl) + : parent(p), parser(pl), type(t) { + } - ssize_t (*read)(struct json_pull *, char *buf, size_t n); - void *source; - char *buffer; - ssize_t buffer_tail; - ssize_t buffer_head; + inline std::string &string(); + inline const std::string &string() const; - json_object *container; - json_object *root; + // large_unsigned() / large_signed() return 0 when the number is not + // held in that representation, the convention callers already expect. + inline double number() const; + inline unsigned long long large_unsigned() const; + inline long long large_signed() const; + inline void set_number(double d); + inline void set_large_unsigned(unsigned long long u); + inline void set_large_signed(long long s); - struct string *number_buffer; -} json_pull; + inline std::vector &array(); + inline const std::vector &array() const; -json_pull *json_begin_file(FILE *f); -json_pull *json_begin_string(const char *s); + inline std::vector &entries(); + inline const std::vector &entries() const; +}; -json_pull *json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source); -void json_end(json_pull *p); +struct json_number : json_object { + enum repr_t { REPR_DOUBLE, + REPR_LARGE_UNSIGNED, + REPR_LARGE_SIGNED }; + + repr_t repr = REPR_DOUBLE; + union value_t { + double d; + unsigned long long u; + long long s; + value_t() + : d(0) { + } + } value; + + json_number() + : json_object(JSON_NUMBER) { + } + json_number(json_object *p, json_pull *pl) + : json_object(JSON_NUMBER, p, pl) { + } +}; + +struct json_string : json_object { + std::string string_value; + + json_string() + : json_object(JSON_STRING) { + } + json_string(json_object *p, json_pull *pl) + : json_object(JSON_STRING, p, pl) { + } +}; + +struct json_array : json_object { + std::vector array_value; + + // 2 slots: coordinate pairs dominate the parse workload. + json_array() + : json_object(JSON_ARRAY) { + array_value.reserve(2); + } + json_array(json_object *p, json_pull *pl) + : json_object(JSON_ARRAY, p, pl) { + array_value.reserve(2); + } +}; + +struct json_hash : json_object { + std::vector entries_value; + + // 4 slots: the typical GeoJSON property hash. + json_hash() + : json_object(JSON_HASH) { + entries_value.reserve(4); + } + json_hash(json_object *p, json_pull *pl) + : json_object(JSON_HASH, p, pl) { + entries_value.reserve(4); + } +}; + +inline std::string &json_object::string() { + assert(type == JSON_STRING); + return static_cast(this)->string_value; +} +inline const std::string &json_object::string() const { + assert(type == JSON_STRING); + return static_cast(this)->string_value; +} + +inline double json_object::number() const { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + switch (n->repr) { + case json_number::REPR_LARGE_UNSIGNED: + return static_cast(n->value.u); + case json_number::REPR_LARGE_SIGNED: + return static_cast(n->value.s); + case json_number::REPR_DOUBLE: + default: + return n->value.d; + } +} +inline unsigned long long json_object::large_unsigned() const { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + return n->repr == json_number::REPR_LARGE_UNSIGNED ? n->value.u : 0; +} +inline long long json_object::large_signed() const { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + return n->repr == json_number::REPR_LARGE_SIGNED ? n->value.s : 0; +} +inline void json_object::set_number(double d) { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + n->repr = json_number::REPR_DOUBLE; + n->value.d = d; +} +inline void json_object::set_large_unsigned(unsigned long long u) { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + n->repr = json_number::REPR_LARGE_UNSIGNED; + n->value.u = u; +} +inline void json_object::set_large_signed(long long s) { + assert(type == JSON_NUMBER); + auto *n = static_cast(this); + n->repr = json_number::REPR_LARGE_SIGNED; + n->value.s = s; +} + +inline std::vector &json_object::array() { + assert(type == JSON_ARRAY); + return static_cast(this)->array_value; +} +inline const std::vector &json_object::array() const { + assert(type == JSON_ARRAY); + return static_cast(this)->array_value; +} + +inline std::vector &json_object::entries() { + assert(type == JSON_HASH); + return static_cast(this)->entries_value; +} +inline const std::vector &json_object::entries() const { + assert(type == JSON_HASH); + return static_cast(this)->entries_value; +} + +inline void json_object_deleter::operator()(json_object *p) const noexcept { + if (p == nullptr) { + return; + } + switch (p->type) { + case JSON_NUMBER: + delete static_cast(p); + break; + case JSON_STRING: + delete static_cast(p); + break; + case JSON_ARRAY: + delete static_cast(p); + break; + case JSON_HASH: + delete static_cast(p); + break; + default: + // JSON_TRUE / JSON_FALSE / JSON_NULL: no extra fields. + delete p; + break; + } +} + +struct json_pull { + const char *error = nullptr; // points at a string literal; no allocation + int line = 1; + + ssize_t (*read)(struct json_pull *, char *buf, size_t n) = nullptr; + void *source = nullptr; + std::vector buffer; + ssize_t buffer_tail = 0; + ssize_t buffer_head = 0; + + // Currently-open containers, innermost last, each with the token it + // expects next. `container` is borrowed; the owner is the surrounding + // container, or `root` for the outermost. + struct parse_frame { + json_object *container; + json_type expect; + }; + std::vector container_stack; + + // Most recently completed top-level value, owned until the next one + // starts parsing or the caller takes it. + json_object_ptr root; + + // Reused across tokens, cleared but not shrunk, so they stop + // reallocating once grown to the largest token seen. + std::string number_buffer; + std::string string_buffer; +}; + +json_pull_ptr json_begin_file(FILE *f); +json_pull_ptr json_begin_string(const char *s); + +json_pull_ptr json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source); + +// Resets the caller's pointer. Optional: the parser frees itself when the +// last json_pull_ptr to it goes away. +void json_end(json_pull_ptr &p); typedef void (*json_separator_callback)(json_type type, json_pull *j, void *state); -json_object *json_read_tree(json_pull *j); -json_object *json_read(json_pull *j); -json_object *json_read_separators(json_pull *j, json_separator_callback cb, void *state); -void json_free(json_object *j); -void json_disconnect(json_object *j); +// The next completed node, borrowed. Valid until the next call that extends +// or trims the parser's tree. nullptr at end of input or on error. +json_object *json_read(json_pull_ptr &j); +json_object *json_read_separators(json_pull_ptr &j, json_separator_callback cb, void *state); +// Drains the next top-level value out of the parser and hands it over. The +// returned tree can outlive the json_pull. See Ownership model for what the +// detach does and does not clear. +json_object_ptr json_read_tree(json_pull_ptr &j); + +// json_free splices `o` out of its parent (if any), or clears the +// parser's root if `o` is the parser's current top-level value, and +// destroys the subtree. After this call, `o` is a dangling pointer +// that must not be used. Safe to call with nullptr. +void json_free(json_object *o); + +// Splices `o` out of whatever owns it and hands it over, same detach as +// json_read_tree. See Ownership model. +json_object_ptr json_disconnect(json_object *o); + +// Borrowed value for `s`. nullptr if `o` is not a hash, `s` is absent, or the +// hash is mid-parse with that value slot unfilled -- a JSON `null` is none of +// those, and comes back as a JSON_NULL node. +json_object *json_hash_get(const json_object_ptr &o, const char *s); json_object *json_hash_get(json_object *o, const char *s); -char *json_stringify(json_object *o); - -#ifdef __cplusplus -} -#endif +std::string json_stringify(const json_object *o); #endif diff --git a/jsontool.cpp b/jsontool.cpp index 5c4e0abe..0afa6276 100644 --- a/jsontool.cpp +++ b/jsontool.cpp @@ -147,17 +147,16 @@ void out(std::string const &s, int type, json_object *properties) { bool found = false; json_object *o = json_hash_get(properties, extract); - if (o != NULL) { + if (o != nullptr) { found = true; - if (o->type == JSON_STRING || o->type == JSON_NUMBER) { - extracted = sort_quote(o->value.string.string); + if (o->type == JSON_STRING) { + extracted = sort_quote(o->string().c_str()); } else { - // Don't really know what to do about sort quoting - // for arbitrary objects - - const char *out = json_stringify(o); - extracted = sort_quote(out); - free((void *) out); + // Numbers, booleans, null, and any other non-string + // values are rendered via json_stringify(); calling + // o->string() here would assert because the type-tagged + // accessor requires JSON_STRING. + extracted = sort_quote(json_stringify(o).c_str()); } } @@ -233,13 +232,13 @@ void join_csv(json_object *j) { } json_object *properties = json_hash_get(j, "properties"); - json_object *key = NULL; + json_object *key = nullptr; - if (properties != NULL) { + if (properties != nullptr) { key = json_hash_get(properties, header[0].c_str()); } - if (key == NULL) { + if (key == nullptr) { static bool warned = false; if (!warned) { fprintf(stderr, "Warning: couldn't find CSV key \"%s\" in JSON\n", header[0].c_str()); @@ -250,13 +249,11 @@ void join_csv(json_object *j) { std::string joinkey; if (key->type == JSON_STRING) { - joinkey = key->value.string.string; + joinkey = key->string(); } else if (key->type == JSON_NUMBER) { - joinkey = milo::dtoa_milo(key->value.number.number); + joinkey = milo::dtoa_milo(key->number()); } else { - const char *s = json_stringify(key); - joinkey = s; - free((void *) s); + joinkey = json_stringify(key); } if (joinkey < prev_joinkey) { @@ -306,14 +303,7 @@ void join_csv(json_object *j) { } if (fields.size() > 0 && joinkey == fields[0]) { - // This knows more about the structure of JSON objects than it ought to - // The 8 is to round up at least as much as SIZE_FOR in json_pull.c - properties->value.object.keys = (json_object **) realloc((void *) properties->value.object.keys, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *)); - properties->value.object.values = (json_object **) realloc((void *) properties->value.object.values, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *)); - if (properties->value.object.keys == NULL || properties->value.object.values == NULL) { - perror("realloc"); - exit(EXIT_MEMORY); - } + properties->entries().reserve(properties->entries().size() + fields.size()); for (size_t i = 1; i < fields.size(); i++) { std::string k = header[i]; @@ -331,46 +321,21 @@ void join_csv(json_object *j) { } if (attr_type != JSON_NULL) { - // This knows more about the structure of JSON objects than it ought to - - json_object *ko = (json_object *) malloc(sizeof(json_object)); - json_object *vo = (json_object *) malloc(sizeof(json_object)); - if (ko == NULL || vo == NULL) { - perror("malloc"); - exit(EXIT_MEMORY); - } - - ko->type = JSON_STRING; - ko->parent = properties; - ko->parser = properties->parser; - - ko->value.string.string = strdup(k.c_str()); - if (ko->value.string.string == NULL) { - perror("strdup"); - exit(EXIT_MEMORY); - } - - vo->type = attr_type; - vo->parent = properties; - vo->parser = properties->parser; + json_object_ptr ko(new json_string(properties, properties->parser)); + ko->string() = k; + json_object_ptr vo; if (attr_type == JSON_STRING) { - vo->value.string.string = strdup(v.c_str()); - if (vo->value.string.string == NULL) { - perror("strdup"); - exit(EXIT_MEMORY); - } + vo = json_object_ptr(new json_string(properties, properties->parser)); + vo->string() = v; } else if (attr_type == JSON_NUMBER) { - vo->value.number.number = atof(v.c_str()); - vo->value.number.large_unsigned = 0; - vo->value.number.large_signed = 0; + vo = json_object_ptr(new json_number(properties, properties->parser)); + vo->set_number(atof(v.c_str())); } else { abort(); } - properties->value.object.keys[properties->value.object.length] = ko; - properties->value.object.values[properties->value.object.length] = vo; - properties->value.object.length++; + properties->entries().push_back({std::move(ko), std::move(vo)}); } } } @@ -383,13 +348,9 @@ struct json_join_action : json_feature_action { join_csv(feature); } - char *s = json_stringify(feature); - out(s, 1, json_hash_get(feature, "properties")); - free(s); + out(json_stringify(feature), 1, json_hash_get(feature, "properties")); } else { - char *s = json_stringify(geometry); - out(s, 2, NULL); - free(s); + out(json_stringify(geometry), 2, nullptr); } return 1; @@ -400,12 +361,11 @@ struct json_join_action : json_feature_action { }; void process(FILE *fp, const char *fname) { - json_pull *jp = json_begin_file(fp); + json_pull_ptr jp = json_begin_file(fp); json_join_action jja; jja.fname = fname; parse_json(&jja, jp); - json_end(jp); } static const struct option long_options[] = { diff --git a/main.cpp b/main.cpp index e084e6e1..5b877d43 100644 --- a/main.cpp +++ b/main.cpp @@ -587,7 +587,7 @@ struct STREAM { } } - json_pull *json_begin() { + json_pull_ptr json_begin() { return ::json_begin(read_stream, this); } }; @@ -1832,7 +1832,7 @@ std::pair read_input(std::vector &sources, char *fname, i // Plain serial reading std::atomic layer_seq(overall_offset); - json_pull *jp = fp->json_begin(); + json_pull_ptr jp = fp->json_begin(); struct serialization_state sst; sst.fname = reading.c_str(); @@ -1859,7 +1859,6 @@ std::pair read_input(std::vector &sources, char *fname, i sst.attribute_types = attribute_types; parse_json(&sst, jp, layer, sources[layer].layer); - json_end(jp); overall_offset = layer_seq; checkdisk(&readers); } @@ -2884,10 +2883,10 @@ void set_attribute_type(std::unordered_map &attribute_types, c void set_attribute_value(const char *arg) { if (*arg == '{') { - json_pull *jp = json_begin_string(arg); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(arg); + json_object_ptr o = json_read_tree(jp); - if (o == NULL) { + if (o == nullptr) { fprintf(stderr, "%s: --set-attribute %s: %s\n", *av, arg, jp->error); exit(EXIT_JSON); } @@ -2897,21 +2896,18 @@ void set_attribute_value(const char *arg) { exit(EXIT_JSON); } - for (size_t i = 0; i < o->value.object.length; i++) { - json_object *k = o->value.object.keys[i]; - json_object *v = o->value.object.values[i]; + for (size_t i = 0; i < o->entries().size(); i++) { + const auto &e = o->entries()[i]; - if (k->type != JSON_STRING) { + if (e.key->type != JSON_STRING) { fprintf(stderr, "%s: --set-attribute %s: key %zu not a string\n", *av, arg, i); exit(EXIT_JSON); } - serial_val val = stringify_value(v, "json", 1, o); - set_attributes.emplace(k->value.string.string, val); + serial_val val = stringify_value(e.value.get(), "json", 1, o.get()); + set_attributes.emplace(e.key->string(), val); } - json_free(o); - json_end(jp); return; } @@ -2936,10 +2932,10 @@ void set_attribute_value(const char *arg) { } void parse_json_source(const char *arg, struct source &src) { - json_pull *jp = json_begin_string(arg); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string(arg); + json_object_ptr o = json_read_tree(jp); - if (o == NULL) { + if (o == nullptr) { fprintf(stderr, "%s: -L%s: %s\n", *av, arg, jp->error); exit(EXIT_JSON); } @@ -2950,30 +2946,27 @@ void parse_json_source(const char *arg, struct source &src) { } json_object *fname = json_hash_get(o, "file"); - if (fname == NULL || fname->type != JSON_STRING) { + if (fname == nullptr || fname->type != JSON_STRING) { fprintf(stderr, "%s: -L%s: requires \"file\": filename\n", *av, arg); exit(EXIT_JSON); } - src.file = std::string(fname->value.string.string); + src.file = fname->string(); json_object *layer = json_hash_get(o, "layer"); - if (layer != NULL && layer->type == JSON_STRING) { - src.layer = std::string(layer->value.string.string); + if (layer != nullptr && layer->type == JSON_STRING) { + src.layer = layer->string(); } json_object *description = json_hash_get(o, "description"); - if (description != NULL && description->type == JSON_STRING) { - src.description = std::string(description->value.string.string); + if (description != nullptr && description->type == JSON_STRING) { + src.description = description->string(); } json_object *format = json_hash_get(o, "format"); - if (format != NULL && format->type == JSON_STRING) { - src.format = std::string(format->value.string.string); + if (format != nullptr && format->type == JSON_STRING) { + src.format = format->string(); } - - json_free(o); - json_end(jp); } static const struct option long_options_orig[] = { @@ -3214,7 +3207,7 @@ int main(int argc, char **argv) { int exclude_all = 0; int read_parallel = 0; int files_open_at_start; - json_object *filter = NULL; + json_object_ptr filter; memsize = calc_memsize(); @@ -3840,7 +3833,7 @@ int main(int argc, char **argv) { auto input_ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_dir, - maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom); + maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter.get(), droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom); ret = std::get<0>(input_ret); @@ -3863,9 +3856,7 @@ int main(int argc, char **argv) { exit(EXIT_IMPOSSIBLE); } - if (filter != NULL) { - json_free(filter); - } + filter.reset(); return ret; } diff --git a/mvt.cpp b/mvt.cpp index 18f239d2..8042db7c 100644 --- a/mvt.cpp +++ b/mvt.cpp @@ -407,7 +407,7 @@ std::string mvt_tile::encode() { std::string feature_string; protozero::pbf_writer feature_writer(feature_string); - if (layers[i].features[f].type >= 0) + if (layers[i].features[f].type >= 0) feature_writer.add_enum(3, layers[i].features[f].type); std::vector sorted_tags = layers[i].features[f].tags; diff --git a/overzoom.cpp b/overzoom.cpp index 5fa4a53f..7209b605 100644 --- a/overzoom.cpp +++ b/overzoom.cpp @@ -263,7 +263,7 @@ int main(int argc, char **argv) { std::string out; { - json_object *json_filter = NULL; + json_object_ptr json_filter; if (filter.size() > 0) { json_filter = parse_filter(filter.c_str()); } @@ -289,7 +289,7 @@ int main(int argc, char **argv) { its.push_back(std::move(t)); } - out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, std::vector(), "", "", SIZE_MAX, std::vector(), deduplicate_by_id); + out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter.get(), preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, std::vector(), "", "", SIZE_MAX, std::vector(), deduplicate_by_id); } FILE *f = fopen(outfile, "wb"); diff --git a/plugin.cpp b/plugin.cpp index cfe06593..17463c05 100644 --- a/plugin.cpp +++ b/plugin.cpp @@ -27,9 +27,7 @@ #include "errors.hpp" #include "thread.hpp" -extern "C" { #include "jsonpull/jsonpull.h" -} #include "plugin.hpp" #include "write_json.hpp" @@ -145,53 +143,55 @@ std::vector parse_layers(int fd, int z, unsigned x, unsigned y, std:: } // Reads from the prefilter -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter, key_pool &key_pool) { +serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool postfilter, key_pool &key_pool) { serial_feature sf; while (1) { + // json_read also returns the intermediate containers of a feature + // still being assembled, so the `continue` paths leave `j` alone; + // freeing one would splice it out of its parent. `j` is only freed + // once it is a complete Feature. json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { + if (j == nullptr) { + if (jp->error != nullptr) { fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error); - if (jp->root != NULL) { - json_context(jp->root); + if (jp->root != nullptr) { + json_context(jp->root.get()); } else { fprintf(stderr, "\n"); } exit(EXIT_JSON); } - json_free(jp->root); + jp->root.reset(); sf.t = -1; return sf; } json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { + if (type == nullptr || type->type != JSON_STRING) { continue; } - if (strcmp(type->value.string.string, "Feature") != 0) { + if (type->string() != "Feature") { continue; } json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { + if (geometry == nullptr) { fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { + if (geometry_type == nullptr) { fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line); json_context(j); exit(EXIT_JSON); @@ -204,7 +204,7 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "Filter output:%d: feature without coordinates array: ", jp->line); json_context(j); exit(EXIT_JSON); @@ -212,12 +212,12 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: int t; for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) { + if (geometry_type->string() == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string); + fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->string().c_str()); json_context(j); exit(EXIT_JSON); } @@ -253,29 +253,29 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: std::string layername = "unknown"; json_object *tippecanoe = json_hash_get(j, "tippecanoe"); - if (tippecanoe != NULL) { + if (tippecanoe != nullptr) { json_object *layer = json_hash_get(tippecanoe, "layer"); - if (layer != NULL && layer->type == JSON_STRING) { - layername = std::string(layer->value.string.string); + if (layer != nullptr && layer->type == JSON_STRING) { + layername = layer->string(); } json_object *index = json_hash_get(tippecanoe, "index"); - if (index != NULL && index->type == JSON_NUMBER) { - sf.index = index->value.number.number; + if (index != nullptr && index->type == JSON_NUMBER) { + sf.index = index->number(); } json_object *sequence = json_hash_get(tippecanoe, "sequence"); - if (sequence != NULL && sequence->type == JSON_NUMBER) { - sf.seq = sequence->value.number.number; + if (sequence != nullptr && sequence->type == JSON_NUMBER) { + sf.seq = sequence->number(); } json_object *extent = json_hash_get(tippecanoe, "extent"); - if (extent != NULL && extent->type == JSON_NUMBER) { - sf.extent = extent->value.number.number; + if (extent != nullptr && extent->type == JSON_NUMBER) { + sf.extent = extent->number(); } json_object *dropped = json_hash_get(tippecanoe, "dropped"); - if (dropped != NULL && dropped->type == JSON_TRUE) { + if (dropped != nullptr && dropped->type == JSON_TRUE) { sf.dropped = FEATURE_DROPPED; // dropped } else { sf.dropped = FEATURE_KEPT; // kept @@ -300,10 +300,10 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } json_object *id = json_hash_get(j, "id"); - if (id != NULL && id->type == JSON_NUMBER) { - sf.id = id->value.number.number; - if (id->value.number.large_unsigned > 0) { - sf.id = id->value.number.large_unsigned; + if (id != nullptr && id->type == JSON_NUMBER) { + sf.id = id->number(); + if (id->large_unsigned() > 0) { + sf.id = id->large_unsigned(); } sf.has_id = true; } @@ -347,18 +347,20 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std:: } } - for (size_t i = 0; i < properties->value.object.length; i++) { - serial_val v = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j); + if (properties->type == JSON_HASH) { + for (const auto &e : properties->entries()) { + serial_val v = stringify_value(e.value.get(), "Filter output", jp->line, j); - // Nulls can be excluded here because the expression evaluation filter - // would have already run before prefiltering + // Nulls can be excluded here because the expression evaluation filter + // would have already run before prefiltering - if (v.type != mvt_null) { - sf.full_keys.push_back(key_pool.pool(std::string(properties->value.object.keys[i]->value.string.string))); - sf.full_values.push_back(v); + if (v.type != mvt_null) { + sf.full_keys.push_back(key_pool.pool(e.key->string())); + sf.full_values.push_back(v); - if (!postfilter) { - add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), v); + if (!postfilter) { + add_to_tilestats(ts->second.tilestats, e.key->string(), v); + } } } } diff --git a/plugin.hpp b/plugin.hpp index fc901daf..5ad99092 100644 --- a/plugin.hpp +++ b/plugin.hpp @@ -1,4 +1,4 @@ struct key_pool; std::vector filter_layers(const char *filter, std::vector &layer, unsigned z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, int extent); void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y); -serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters, key_pool &key_pool); +serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, std::vector> *layermaps, size_t tiling_seg, std::vector> *layer_unmaps, bool filters, key_pool &key_pool); diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index e86e538b..4d842869 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -397,9 +397,9 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { exit(EXIT_OPEN); } - json_pull *jp = json_begin_string(decompressed_json.c_str()); - json_object *o = json_read_tree(jp); - if (o == NULL) { + json_pull_ptr jp = json_begin_string(decompressed_json.c_str()); + json_object_ptr o = json_read_tree(jp); + if (o == nullptr) { fprintf(stderr, "%s: metadata parsing error: %s\n", fname, jp->error); exit(EXIT_JSON); } @@ -415,45 +415,50 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) { state.nospace = true; state.json_write_hash(); - for (size_t i = 0; i < o->value.object.length; i++) { - const char *key = o->value.object.keys[i]->value.string.string; - if (strcmp(key, "vector_layers") == 0 && o->value.object.values[i]->type == JSON_ARRAY) { + for (const auto &e : o->entries()) { + // Check before reading, since string() asserts on the type. + if (e.key->type != JSON_STRING) { + fprintf(stderr, "%s: non-string key in metadata\n", fname); + continue; + } + + const std::string &key = e.key->string(); + if (key == "vector_layers" && e.value->type == JSON_ARRAY) { has_json = true; state.nospace = true; state.json_write_string("vector_layers"); state.nospace = true; - state.json_write_json(json_stringify(o->value.object.values[i])); - } else if (strcmp(key, "tilestats") == 0 && o->value.object.values[i]->type == JSON_HASH) { + state.json_write_json(json_stringify(e.value.get())); + } else if (key == "tilestats" && e.value->type == JSON_HASH) { has_json = true; state.nospace = true; state.json_write_string("tilestats"); state.nospace = true; - state.json_write_json(json_stringify(o->value.object.values[i])); - } else if (strcmp(key, "strategies") == 0 && o->value.object.values[i]->type == JSON_ARRAY) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(o->value.object.values[i])); + state.json_write_json(json_stringify(e.value.get())); + } else if (key == "strategies" && e.value->type == JSON_ARRAY) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(e.value.get()).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", key, err); + fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); - } else if (strcmp(key, "tippecanoe_decisions") == 0 && o->value.object.values[i]->type == JSON_HASH) { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(o->value.object.values[i])); + } else if (key == "tippecanoe_decisions" && e.value->type == JSON_HASH) { + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(e.value.get()).c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", key, err); + fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); - } else if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) { - fprintf(stderr, "%s\n", key); + } else if (e.value->type != JSON_STRING) { + fprintf(stderr, "%s\n", key.c_str()); fprintf(stderr, "%s: non-string in metadata\n", fname); } else { - sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key, o->value.object.values[i]->value.string.string); + sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key.c_str(), e.value->string().c_str()); if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "set %s in metadata: %s\n", key, err); + fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err); } sqlite3_free(sql); } } - json_end(jp); state.nospace = true; state.json_end_hash(); diff --git a/read_json.cpp b/read_json.cpp index 63329acb..ec0b1cfd 100644 --- a/read_json.cpp +++ b/read_json.cpp @@ -43,18 +43,18 @@ int mb_geometry[GEOM_TYPES] = { }; void json_context(json_object *j) { - char *s = json_stringify(j); + std::string s = json_stringify(j); - if (strlen(s) >= 500) { - snprintf(s + 497, strlen(s) + 1 - 497, "..."); + if (s.size() >= 500) { + s.resize(497); + s.append("..."); } - fprintf(stderr, "in JSON object %s\n", s); - free(s); // stringify + fprintf(stderr, "in JSON object %s\n", s.c_str()); } void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature) { - if (j == NULL || j->type != JSON_ARRAY) { + if (j == nullptr || j->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: expected array for geometry type %d: ", fname, line, t); json_context(feature); return; @@ -63,7 +63,7 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char * int within = geometry_within[t]; if (within >= 0) { size_t i; - for (i = 0; i < j->value.array.length; i++) { + for (i = 0; i < j->array().size(); i++) { if (within == GEOM_POINT) { if (i == 0 || mb_geometry[t] == VT_POINT) { op = VT_MOVETO; @@ -72,16 +72,16 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char * } } - parse_coordinates(within, j->value.array.array[i], out, op, fname, line, feature); + parse_coordinates(within, j->array()[i].get(), out, op, fname, line, feature); } } else { - if (j->value.array.length >= 2 && j->value.array.array[0]->type == JSON_NUMBER && j->value.array.array[1]->type == JSON_NUMBER) { + if (j->array().size() >= 2 && j->array()[0]->type == JSON_NUMBER && j->array()[1]->type == JSON_NUMBER) { long long x, y; - double lon = j->value.array.array[0]->value.number.number; - double lat = j->value.array.array[1]->value.number.number; + double lon = j->array()[0]->number(); + double lat = j->array()[1]->number(); projection->project(lon, lat, 32, &x, &y); - if (j->value.array.length > 2) { + if (j->array().size() > 2) { static int warned = 0; if (!warned) { @@ -124,12 +124,12 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char * serial_val stringify_value(json_object *value, const char *reading, int line, json_object *feature) { serial_val sv; - if (value != NULL) { + if (value != nullptr) { int vt = value->type; if (vt == JSON_STRING) { sv.type = mvt_string; - sv.s = value->value.string.string; + sv.s = value->string(); std::string err = check_utf8(sv.s); if (err.size() > 0) { @@ -140,12 +140,12 @@ serial_val stringify_value(json_object *value, const char *reading, int line, js } else if (vt == JSON_NUMBER) { sv.type = mvt_double; - if (value->value.number.large_unsigned != 0) { - sv.s = std::to_string(value->value.number.large_unsigned); - } else if (value->value.number.large_signed != 0) { - sv.s = std::to_string(value->value.number.large_signed); + if (value->large_unsigned() != 0) { + sv.s = std::to_string(value->large_unsigned()); + } else if (value->large_signed() != 0) { + sv.s = std::to_string(value->large_signed()); } else { - sv.s = milo::dtoa_milo(value->value.number.number); + sv.s = milo::dtoa_milo(value->number()); } } else if (vt == JSON_TRUE) { sv.type = mvt_bool; @@ -158,9 +158,7 @@ serial_val stringify_value(json_object *value, const char *reading, int line, js sv.s = "null"; } else { sv.type = mvt_string; - const char *v = json_stringify(value); - sv.s = std::string(v); - free((void *) v); // stringify + sv.s = json_stringify(value); } } @@ -178,10 +176,10 @@ static std::vector to_feature(drawvec &geom) { return out; } -std::pair parse_geometry(json_object *geometry, json_pull *jp, json_object *j, +std::pair parse_geometry(json_object *geometry, json_pull_ptr &jp, json_object *j, int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style) { json_object *geometry_type = json_hash_get(geometry, "type"); - if (geometry_type == NULL) { + if (geometry_type == nullptr) { fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line); json_context(j); exit(EXIT_JSON); @@ -194,7 +192,7 @@ std::pair parse_geometry(json_object *geometry, json_pull *jp, jso } json_object *coordinates = json_hash_get(geometry, "coordinates"); - if (coordinates == NULL || coordinates->type != JSON_ARRAY) { + if (coordinates == nullptr || coordinates->type != JSON_ARRAY) { fprintf(stderr, "Filter output:%d: geometry without coordinates array: ", jp->line); json_context(j); exit(EXIT_JSON); @@ -202,12 +200,12 @@ std::pair parse_geometry(json_object *geometry, json_pull *jp, jso int t; for (t = 0; t < GEOM_TYPES; t++) { - if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) { + if (geometry_type->string() == geometry_names[t]) { break; } } if (t >= GEOM_TYPES) { - fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string); + fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->string().c_str()); json_context(j); exit(EXIT_JSON); } @@ -305,47 +303,47 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int std::map ret; std::shared_ptr tile_stringpool = std::make_shared(); - json_pull *jp = json_begin_file(fp); + json_pull_ptr jp = json_begin_file(fp); while (1) { json_object *j = json_read(jp); - if (j == NULL) { - if (jp->error != NULL) { + if (j == nullptr) { + if (jp->error != nullptr) { fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error); - if (jp->root != NULL) { - json_context(jp->root); + if (jp->root != nullptr) { + json_context(jp->root.get()); } else { fprintf(stderr, "\n"); } exit(EXIT_JSON); } - json_free(jp->root); + jp->root.reset(); break; } + // Only complete Features are freed; see plugin.cpp::parse_feature. json_object *type = json_hash_get(j, "type"); - if (type == NULL || type->type != JSON_STRING) { + if (type == nullptr || type->type != JSON_STRING) { continue; } - if (strcmp(type->value.string.string, "Feature") != 0) { + if (type->string() != "Feature") { continue; } json_object *properties = json_hash_get(j, "properties"); - if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { + if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) { fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } std::string layername = "unknown"; json_object *tippecanoe = json_hash_get(j, "tippecanoe"); - json_object *layer = NULL; - if (tippecanoe != NULL) { + json_object *layer = nullptr; + if (tippecanoe != nullptr) { layer = json_hash_get(tippecanoe, "layer"); - if (layer != NULL && layer->type == JSON_STRING) { - layername = std::string(layer->value.string.string); + if (layer != nullptr && layer->type == JSON_STRING) { + layername = layer->string(); } } @@ -360,10 +358,9 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int auto l = ret.find(layername); json_object *geometry = json_hash_get(j, "geometry"); - if (geometry == NULL) { + if (geometry == nullptr) { fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line); json_context(j); - json_free(j); exit(EXIT_JSON); } @@ -378,23 +375,25 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int feature.geometry = to_feature(dv); json_object *id = json_hash_get(j, "id"); - if (id != NULL && id->type == JSON_NUMBER) { - feature.id = id->value.number.number; - if (id->value.number.large_unsigned > 0) { - feature.id = id->value.number.large_unsigned; + if (id != nullptr && id->type == JSON_NUMBER) { + feature.id = id->number(); + if (id->large_unsigned() > 0) { + feature.id = id->large_unsigned(); } feature.has_id = true; } - for (size_t i = 0; i < properties->value.object.length; i++) { - serial_val sv = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j); + if (properties->type == JSON_HASH) { + for (const auto &e : properties->entries()) { + serial_val sv = stringify_value(e.value.get(), "Filter output", jp->line, j); - // Nulls can be excluded here because this is the postfilter - // and it is nearly time to create the vector representation + // Nulls can be excluded here because this is the postfilter + // and it is nearly time to create the vector representation - if (sv.type != mvt_null) { - mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); - l->second.tag(feature, std::string(properties->value.object.keys[i]->value.string.string), v); + if (sv.type != mvt_null) { + mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool); + l->second.tag(feature, e.key->string(), v); + } } } @@ -404,8 +403,6 @@ std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int json_free(j); } - json_end(jp); - std::vector final; for (auto a : ret) { final.push_back(a.second); diff --git a/read_json.hpp b/read_json.hpp index a5d5d7b2..f254e354 100644 --- a/read_json.hpp +++ b/read_json.hpp @@ -12,7 +12,7 @@ extern int mb_geometry[GEOM_TYPES]; void json_context(json_object *j); void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature); -std::pair parse_geometry(json_object *geometry, json_pull *jp, json_object *j, +std::pair parse_geometry(json_object *geometry, json_pull_ptr &jp, json_object *j, int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style); std::vector parse_layers(FILE *fp, int z, unsigned x, unsigned y, int extent, bool fix_longitudes); diff --git a/tile-join.cpp b/tile-join.cpp index 017e04b8..84666840 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -336,7 +336,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map> &tasks, std::vector< } void handle_strategies(const unsigned char *s, std::vector *st) { - json_pull *jp = json_begin_string((const char *) s); - json_object *o = json_read_tree(jp); + json_pull_ptr jp = json_begin_string((const char *) s); + json_object_ptr o = json_read_tree(jp); - if (o != NULL && o->type == JSON_ARRAY) { - for (size_t i = 0; i < o->value.array.length; i++) { - json_object *h = o->value.array.array[i]; + if (o != nullptr && o->type == JSON_ARRAY) { + for (size_t i = 0; i < o->array().size(); i++) { + const json_object_ptr &h = o->array()[i]; if (h->type == JSON_HASH) { - for (size_t j = 0; j < h->value.object.length; j++) { - json_object *k = h->value.object.keys[j]; - json_object *v = h->value.object.values[j]; + for (size_t j = 0; j < h->entries().size(); j++) { + const auto &kv = h->entries()[j]; - if (k->type != JSON_STRING) { + if (kv.key->type != JSON_STRING) { fprintf(stderr, "Key %zu of %zu is not a string: %s\n", j, i, s); - } else if (v->type != JSON_NUMBER) { + } else if (kv.value->type != JSON_NUMBER) { fprintf(stderr, "Value %zu of %zu is not a number: %s\n", j, i, s); } else { if (i >= st->size()) { st->resize(i + 1); } - if (strcmp(k->value.string.string, "dropped_by_rate") == 0) { - (*st)[i].dropped_by_rate += v->value.number.number; - } else if (strcmp(k->value.string.string, "dropped_by_gamma") == 0) { - (*st)[i].dropped_by_gamma += v->value.number.number; - } else if (strcmp(k->value.string.string, "dropped_as_needed") == 0) { - (*st)[i].dropped_as_needed += v->value.number.number; - } else if (strcmp(k->value.string.string, "coalesced_as_needed") == 0) { - (*st)[i].coalesced_as_needed += v->value.number.number; - } else if (strcmp(k->value.string.string, "truncated_zooms") == 0) { - (*st)[i].truncated_zooms += v->value.number.number; - } else if (strcmp(k->value.string.string, "detail_reduced") == 0) { - (*st)[i].detail_reduced += v->value.number.number; - } else if (strcmp(k->value.string.string, "tiny_polygons") == 0) { - (*st)[i].tiny_polygons += v->value.number.number; - } else if (strcmp(k->value.string.string, "tile_size_desired") == 0) { - (*st)[i].tile_size += v->value.number.number; - } else if (strcmp(k->value.string.string, "feature_count_desired") == 0) { - (*st)[i].feature_count += v->value.number.number; + const std::string &key = kv.key->string(); + if (key == "dropped_by_rate") { + (*st)[i].dropped_by_rate += kv.value->number(); + } else if (key == "dropped_by_gamma") { + (*st)[i].dropped_by_gamma += kv.value->number(); + } else if (key == "dropped_as_needed") { + (*st)[i].dropped_as_needed += kv.value->number(); + } else if (key == "coalesced_as_needed") { + (*st)[i].coalesced_as_needed += kv.value->number(); + } else if (key == "truncated_zooms") { + (*st)[i].truncated_zooms += kv.value->number(); + } else if (key == "detail_reduced") { + (*st)[i].detail_reduced += kv.value->number(); + } else if (key == "tiny_polygons") { + (*st)[i].tiny_polygons += kv.value->number(); + } else if (key == "tile_size_desired") { + (*st)[i].tile_size += kv.value->number(); + } else if (key == "feature_count_desired") { + (*st)[i].feature_count += kv.value->number(); } } } @@ -1011,22 +1011,19 @@ void handle_strategies(const unsigned char *s, std::vector *st) { fprintf(stderr, "Element %zu is not a hash: %s\n", i, s); } } - json_free(o); } - - json_end(jp); } void handle_vector_layers(json_object *vector_layers, std::map &layermap, std::map &attribute_descriptions) { - if (vector_layers != NULL && vector_layers->type == JSON_ARRAY) { - for (size_t i = 0; i < vector_layers->value.array.length; i++) { - if (vector_layers->value.array.array[i]->type == JSON_HASH) { - json_object *id = json_hash_get(vector_layers->value.array.array[i], "id"); - json_object *desc = json_hash_get(vector_layers->value.array.array[i], "description"); + if (vector_layers != nullptr && vector_layers->type == JSON_ARRAY) { + for (size_t i = 0; i < vector_layers->array().size(); i++) { + if (vector_layers->array()[i]->type == JSON_HASH) { + json_object *id = json_hash_get(vector_layers->array()[i].get(), "id"); + json_object *desc = json_hash_get(vector_layers->array()[i].get(), "description"); - if (id != NULL && desc != NULL && id->type == JSON_STRING && desc->type == JSON_STRING) { - std::string sid = id->value.string.string; - std::string sdesc = desc->value.string.string; + if (id != nullptr && desc != nullptr && id->type == JSON_STRING && desc->type == JSON_STRING) { + const std::string &sid = id->string(); + const std::string &sdesc = desc->string(); if (sdesc.size() != 0) { auto f = layermap.find(sid); @@ -1036,17 +1033,18 @@ void handle_vector_layers(json_object *vector_layers, std::mapvalue.array.array[i], "fields"); - if (fields != NULL && fields->type == JSON_HASH) { - for (size_t j = 0; j < fields->value.object.length; j++) { - if (fields->value.object.keys[j]->type == JSON_STRING && fields->value.object.values[j]->type) { - const char *desc2 = fields->value.object.values[j]->value.string.string; + json_object *fields = json_hash_get(vector_layers->array()[i].get(), "fields"); + if (fields != nullptr && fields->type == JSON_HASH) { + for (const auto &e : fields->entries()) { + if (e.key != nullptr && e.key->type == JSON_STRING && + e.value != nullptr && e.value->type == JSON_STRING) { + const std::string &desc2 = e.value->string(); - if (strcmp(desc2, "Number") != 0 && - strcmp(desc2, "String") != 0 && - strcmp(desc2, "Boolean") != 0 && - strcmp(desc2, "Mixed") != 0) { - attribute_descriptions.insert(std::pair(fields->value.object.keys[j]->value.string.string, desc2)); + if (desc2 != "Number" && + desc2 != "String" && + desc2 != "Boolean" && + desc2 != "Mixed") { + attribute_descriptions.insert(std::pair(e.key->string(), desc2)); } } } @@ -1206,17 +1204,14 @@ void decode(struct tileset_reader *readers, std::maptype == JSON_HASH) { + if (o != nullptr && o->type == JSON_HASH) { json_object *vector_layers = json_hash_get(o, "vector_layers"); handle_vector_layers(vector_layers, layermap, attribute_descriptions); - json_free(o); } - - json_end(jp); } } @@ -1340,7 +1335,7 @@ int main(int argc, char **argv) { int force = 0; int ifmatched = 0; int filearg = 0; - json_object *filter = NULL; + json_object_ptr filter; std::string join_sqlite_fname; @@ -1621,7 +1616,7 @@ int main(int argc, char **argv) { std::string generator_options; std::vector strategies; - decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies); + decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter.get(), attribute_descriptions, generator_options, &strategies); if (set_attribution.size() != 0) { attribution = set_attribution; @@ -1673,9 +1668,7 @@ int main(int argc, char **argv) { mbtiles_close(outdb, argv[0]); } - if (filter != NULL) { - json_free(filter); - } + filter.reset(); if (pmtiles_has_suffix(out_mbtiles)) { mbtiles_map_image_to_pmtiles(out_mbtiles, m, !pC, quiet, false); diff --git a/tile.cpp b/tile.cpp index 6d98ecdc..f601a235 100644 --- a/tile.cpp +++ b/tile.cpp @@ -48,11 +48,7 @@ #include "attribute.hpp" #include "thread.hpp" #include "shared_borders.hpp" - -extern "C" { #include "jsonpull/jsonpull.h" -} - #include "plugin.hpp" #define CMD_BITS 3 @@ -827,7 +823,7 @@ static double choose_minattribute(std::vector &attribute_values, double if (descending) { // For descending: drop features > threshold, keep features <= threshold // ix points at the last value to keep - size_t ix = (size_t)((attribute_values.size() - 1) * f); + size_t ix = (size_t) ((attribute_values.size() - 1) * f); while (ix > 0 && attribute_values[ix] >= existing_attribute) { ix--; } @@ -840,7 +836,7 @@ static double choose_minattribute(std::vector &attribute_values, double } else { // For ascending: drop features < threshold, keep features >= threshold // ix points at the first value to keep - size_t ix = (size_t)ceil((double)(attribute_values.size() - 1) * (1 - f)); + size_t ix = (size_t) ceil((double) (attribute_values.size() - 1) * (1 - f)); if (ix >= attribute_values.size()) { ix = attribute_values.size() - 1; } @@ -1796,7 +1792,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch pthread_t prefilter_writer; run_prefilter_args rpa; // here so it stays in scope until joined FILE *prefilter_read_fp = NULL; - json_pull *prefilter_jp = NULL; + json_pull_ptr prefilter_jp; if (z < minzoom) { prefilter = NULL; @@ -2132,8 +2128,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (attr_valid) { add_sample_to(attribute_values, attr_numeric, attribute_values_increment, seq); bool should_drop = arg->drop_by_attribute_descending - ? (minattribute != HUGE_VAL && attr_numeric > minattribute) - : (minattribute != -HUGE_VAL && attr_numeric < minattribute); + ? (minattribute != HUGE_VAL && attr_numeric > minattribute) + : (minattribute != -HUGE_VAL && attr_numeric < minattribute); if (should_drop) { can_stop_early = false; if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum, key_pool)) { @@ -2801,7 +2797,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } } else if (additional[A_DROP_BY_ATTRIBUTE_AS_NEEDED]) { minattribute_fraction = minattribute_fraction * - adjusted_max_tile_features / adjusted_feature_count * 0.75; + adjusted_max_tile_features / adjusted_feature_count * 0.75; if (minattribute_fraction > 0.80) { if (!quiet) { fprintf(stderr, @@ -3520,8 +3516,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std:: again = true; } bool attr_propagate = drop_by_attribute_descending - ? args[thread].minattribute_out < zoom_minattribute - : args[thread].minattribute_out > zoom_minattribute; + ? args[thread].minattribute_out < zoom_minattribute + : args[thread].minattribute_out > zoom_minattribute; if (attr_propagate) { zoom_minattribute = args[thread].minattribute_out; again = true; diff --git a/tile.hpp b/tile.hpp index 8f266c0b..8b72e3fb 100644 --- a/tile.hpp +++ b/tile.hpp @@ -62,7 +62,7 @@ struct strategy { // long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic *strategy); -int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending); +int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/unit.cpp b/unit.cpp index dc752152..7a04e20b 100644 --- a/unit.cpp +++ b/unit.cpp @@ -6,6 +6,7 @@ #include "mvt.hpp" #include "projection.hpp" #include "geometry.hpp" +#include "jsonpull/jsonpull.h" #include #include @@ -137,6 +138,364 @@ TEST_CASE("line_is_too_small") { REQUIRE(line_is_too_small(dv, 0, 10)); } +// A high surrogate followed by a non-surrogate used to be combined into one +// code point, because the range check tested the outer-loop byte instead of +// the parsed code unit. The stale surrogate should come out as standalone +// CESU-8, then U+E000 encoded normally. +TEST_CASE("jsonpull surrogate-pair regression", "[jsonpull][surrogate]") { + json_pull_ptr jp = json_begin_string("\"\\uD83D\\uE000\""); + json_object_ptr o = json_read_tree(jp); + + REQUIRE(jp->error == nullptr); + REQUIRE(o != nullptr); + REQUIRE(o->type == JSON_STRING); + + const std::string expected = "\xED\xA0\xBD\xEE\x80\x80"; + REQUIRE(o->string() == expected); + + // Sanity check: the buggy output (a single 4-byte UTF-8 sequence for + // U+1F400) must not be what we got. + const std::string buggy = "\xF0\x9F\x90\x80"; + REQUIRE(o->string() != buggy); +} + +// What geojson-loop does: free each feature once serialized, so they do not +// accumulate while the rest of the document is parsed. +// +// This shape cannot catch the array-splicing bug, at any size: json_read hands +// back each container as it completes, so the node freed here is always the +// last-added element of its parent, which the old memmove got right by moving +// zero bytes. "json_free prunes a non-final element" below covers that. +TEST_CASE("json_free prunes a subtree from its parent", "[jsonpull][memory]") { + json_pull_ptr jp = json_begin_string("[[1, 2], [3, 4], [5, 6]]"); + + json_object *outer = nullptr; + int arrays_seen = 0; + + json_object *j; + while ((j = json_read(jp)) != nullptr) { + if (j->type != JSON_ARRAY) { + continue; + } + arrays_seen++; + if (arrays_seen == 2) { + // This is [3, 4]; verify, then ask the parser to drop it. + REQUIRE(j->array().size() == 2); + REQUIRE(j->array()[0]->number() == 3); + REQUIRE(j->array()[1]->number() == 4); + json_free(j); + } else if (j->parent == nullptr) { + // The completed outer array; the parser still owns it + // via jp->root, so the borrowed pointer stays valid. + outer = j; + break; + } + } + + REQUIRE(outer != nullptr); + REQUIRE(outer->type == JSON_ARRAY); + REQUIRE(outer->array().size() == 2); + + // First surviving element: [1, 2]. + REQUIRE(outer->array()[0]->type == JSON_ARRAY); + REQUIRE(outer->array()[0]->array().size() == 2); + REQUIRE(outer->array()[0]->array()[0]->number() == 1); + REQUIRE(outer->array()[0]->array()[1]->number() == 2); + + // Second surviving element (previously third): [5, 6]. + REQUIRE(outer->array()[1]->type == JSON_ARRAY); + REQUIRE(outer->array()[1]->array().size() == 2); + REQUIRE(outer->array()[1]->array()[0]->number() == 5); + REQUIRE(outer->array()[1]->array()[1]->number() == 6); +} + +// A top-level value has no parent, so json_free has to drop the parser's +// reference instead. jp->root being empty afterwards is also proof the subtree +// was destroyed, since jp->root was its only owner. +TEST_CASE("json_free releases a top-level value held by the parser", "[jsonpull][memory]") { + json_pull_ptr jp = json_begin_string(R"({"a": 1, "b": [2, 3]})"); + + // json_read streams atoms first (1, 2, 3, [2,3], ...); the top-level + // hash is returned by the final `}` token. + json_object *top = nullptr; + json_object *j; + while ((j = json_read(jp)) != nullptr) { + if (j->parent == nullptr) { + top = j; + break; + } + } + + REQUIRE(top != nullptr); + REQUIRE(top->type == JSON_HASH); + REQUIRE(jp->root.get() == top); + + json_free(top); + // top is dangling now; do not dereference. + + REQUIRE(jp->root == nullptr); +} + +// Nothing in tippecanoe calls json_disconnect today -- the filter loaders get +// the same guarantee from json_read_tree -- so cover it here rather than leave +// a documented ownership primitive untested. +TEST_CASE("json_disconnect hands a subtree to the caller", "[jsonpull][ownership]") { + json_object_ptr taken; + json_object *outer = nullptr; + + json_pull_ptr jp = json_begin_string("[[1, 2], [3, 4]]"); + + int arrays_seen = 0; + json_object *j; + while ((j = json_read(jp)) != nullptr) { + if (j->type != JSON_ARRAY) { + continue; + } + arrays_seen++; + if (arrays_seen == 2) { + // This is [3, 4]; take it away from the enclosing array. + taken = json_disconnect(j); + REQUIRE(taken != nullptr); + REQUIRE(taken.get() == j); + } else if (j->parent == nullptr) { + outer = j; + break; + } + } + + // The outer array is left holding only the element we didn't take, + // and the parser is still the owner of it. + REQUIRE(outer != nullptr); + REQUIRE(jp->root.get() == outer); + REQUIRE(outer->array().size() == 1); + REQUIRE(outer->array()[0]->array().size() == 2); + REQUIRE(outer->array()[0]->array()[0]->number() == 1); + REQUIRE(outer->array()[0]->array()[1]->number() == 2); + + // The detached subtree holds no pointers into the parser... + REQUIRE(taken->parser == nullptr); + REQUIRE(taken->array()[0]->parser == nullptr); + REQUIRE(taken->array()[1]->parser == nullptr); + + // ...and its root no longer points out at the array it was spliced + // from, but the parent links *within* it are left intact so the tree + // stays navigable upwards. + REQUIRE(taken->parent == nullptr); + REQUIRE(taken->array()[0]->parent == taken.get()); + REQUIRE(taken->array()[1]->parent == taken.get()); + + // So the subtree stays valid once the parser, and the tree the parser + // still owns, are destroyed. + jp.reset(); + // outer is dangling now; do not dereference. + + REQUIRE(taken->type == JSON_ARRAY); + REQUIRE(taken->array().size() == 2); + REQUIRE(taken->array()[0]->number() == 3); + REQUIRE(taken->array()[1]->number() == 4); +} + +// json_free finds a node's owner through o->parent, so clearing the interior +// parent links on detach would make this a silent no-op. +TEST_CASE("json_free prunes an interior node of a detached tree", "[jsonpull][ownership]") { + json_pull_ptr jp = json_begin_string("[[1, 2], [3, 4], [5, 6]]"); + json_object_ptr tree = json_read_tree(jp); + + REQUIRE(tree != nullptr); + REQUIRE(tree->type == JSON_ARRAY); + REQUIRE(tree->array().size() == 3); + + // Destroy the parser first, so this is unambiguously operating on a + // tree that no longer has one. + jp.reset(); + REQUIRE(tree->parser == nullptr); + + json_object *drop = tree->array()[1].get(); + REQUIRE(drop->array()[0]->number() == 3); + REQUIRE(drop->parent == tree.get()); + + json_free(drop); + // drop is dangling now; do not dereference. + + // Had the parent links been cleared on detach, json_free would not + // have found an owner to splice the node out of, and the array would + // still have three elements. + REQUIRE(tree->array().size() == 2); + REQUIRE(tree->array()[0]->array()[0]->number() == 1); + REQUIRE(tree->array()[1]->array()[0]->number() == 5); +} + +// Not a removal: the key stays with a JSON_NULL stand-in so the surrounding +// pairs keep their alignment, and the entry goes only when both halves do. +TEST_CASE("json_free of a hash value leaves a null placeholder", "[jsonpull][ownership]") { + json_pull_ptr jp = json_begin_string(R"({"keep": 1, "drop": [2, 3]})"); + json_object_ptr tree = json_read_tree(jp); + + REQUIRE(tree != nullptr); + REQUIRE(tree->entries().size() == 2); + + json_object *drop = json_hash_get(tree, "drop"); + REQUIRE(drop != nullptr); + REQUIRE(drop->type == JSON_ARRAY); + + json_free(drop); + // drop is dangling now; do not dereference. + + // The key survives, now paired with a null rather than the array. + REQUIRE(tree->entries().size() == 2); + json_object *after = json_hash_get(tree, "drop"); + REQUIRE(after != nullptr); + REQUIRE(after->type == JSON_NULL); + + // The neighbouring pair is untouched. + REQUIRE(json_hash_get(tree, "keep") != nullptr); + REQUIRE(json_hash_get(tree, "keep")->number() == 1); +} + +// The array-splicing fix. The old code passed an element count to memmove +// where a byte count was wanted, so pruning element 0 of eight left +// arr[0] == arr[1] -- one node owned twice -- and dropped the last element. +// Asserting each survivor's identity is what discriminates; checking only the +// size would not. +TEST_CASE("json_free prunes a non-final element", "[jsonpull][ownership]") { + json_pull_ptr jp = json_begin_string("[[1], [2], [3], [4], [5], [6], [7], [8]]"); + json_object_ptr tree = json_read_tree(jp); + + REQUIRE(tree != nullptr); + REQUIRE(tree->type == JSON_ARRAY); + REQUIRE(tree->array().size() == 8); + + json_free(tree->array()[0].get()); + + // Every survivor keeps its identity, in order, and nothing is aliased. + REQUIRE(tree->array().size() == 7); + for (size_t i = 0; i < tree->array().size(); i++) { + json_object *e = tree->array()[i].get(); + REQUIRE(e->type == JSON_ARRAY); + REQUIRE(e->array().size() == 1); + REQUIRE(e->array()[0]->number() == (double) (i + 2)); + + if (i + 1 < tree->array().size()) { + REQUIRE(e != tree->array()[i + 1].get()); + } + } +} + +// The mirror of the value case above. +TEST_CASE("json_free of a hash key, then of both halves", "[jsonpull][ownership]") { + json_pull_ptr jp = json_begin_string(R"({"a": 1, "b": 2, "c": 3})"); + json_object_ptr tree = json_read_tree(jp); + + REQUIRE(tree != nullptr); + REQUIRE(tree->entries().size() == 3); + + // Free the key of the middle pair. The entry stays, with a null key, + // and its value is still reachable positionally. + json_free(tree->entries()[1].key.get()); + + REQUIRE(tree->entries().size() == 3); + REQUIRE(tree->entries()[1].key->type == JSON_NULL); + REQUIRE(tree->entries()[1].value->number() == 2); + + // The neighbours are untouched, and the now-keyless pair is no longer + // findable by name. + REQUIRE(json_hash_get(tree, "b") == nullptr); + REQUIRE(json_hash_get(tree, "a")->number() == 1); + REQUIRE(json_hash_get(tree, "c")->number() == 3); + + // Freeing the other half too retires the whole entry. + json_free(tree->entries()[1].value.get()); + + REQUIRE(tree->entries().size() == 2); + REQUIRE(json_hash_get(tree, "a")->number() == 1); + REQUIRE(json_hash_get(tree, "c")->number() == 3); +} + +// What the filter loaders and -L / -E do. Each detached tree has to survive +// the next json_read_tree call and the parser's destruction. +TEST_CASE("repeated json_read_tree on a line-delimited stream", "[jsonpull][ownership]") { + json_pull_ptr jp = json_begin_string("{\"n\": 1}\n{\"n\": 2}\n{\"n\": 3}\n"); + + std::vector trees; + for (int i = 0; i < 3; i++) { + json_object_ptr t = json_read_tree(jp); + REQUIRE(t != nullptr); + REQUIRE(t->type == JSON_HASH); + // Reading the next tree must not disturb the ones already taken. + REQUIRE(json_hash_get(t, "n")->number() == i + 1); + trees.push_back(std::move(t)); + } + + REQUIRE(json_read_tree(jp) == nullptr); + + // All three outlive the parser they came from. + jp.reset(); + for (int i = 0; i < 3; i++) { + REQUIRE(trees[i]->parser == nullptr); + REQUIRE(json_hash_get(trees[i], "n")->number() == i + 1); + } +} + +// What json_context() prints on the error paths: a hash whose last key has no +// value yet renders that slot as "...". +TEST_CASE("json_stringify of a partially-parsed tree", "[jsonpull][stringify]") { + json_pull_ptr jp = json_begin_string("{\"a\": [1, 2], \"b\":"); + + // Read until the parser runs out of input mid-hash. + while (json_read(jp) != nullptr) { + ; + } + REQUIRE(jp->error != nullptr); + REQUIRE(jp->root != nullptr); + + std::string s = json_stringify(jp->root.get()); + REQUIRE(s == "{\"a\":[1,2],\"b\":...}"); +} + +// Values are std::string now, so an embedded NUL is legal and stringify has to +// walk past it rather than stop the way a c_str() loop would. +TEST_CASE("json_stringify keeps text after an embedded NUL", "[jsonpull][stringify]") { + json_pull_ptr jp = json_begin_string("\"a\\u0000b\""); + json_object_ptr o = json_read_tree(jp); + + REQUIRE(o != nullptr); + REQUIRE(o->type == JSON_STRING); + REQUIRE(o->string().size() == 3); + + std::string s = json_stringify(o.get()); + REQUIRE(s == "\"a\\u0000b\""); +} + +// U+FFFF used to fall through the `< 0xFFFF` test into the four-byte branch +// and come out as the overlong F0 8F BF BF, which check_utf8 does not catch. +TEST_CASE("jsonpull encodes U+FFFF as three bytes", "[jsonpull][utf8]") { + json_pull_ptr jp = json_begin_string("\"a\\uFFFFb\""); + json_object_ptr o = json_read_tree(jp); + + REQUIRE(jp->error == nullptr); + REQUIRE(o != nullptr); + REQUIRE(o->type == JSON_STRING); + + REQUIRE(o->string() == + "a\xEF\xBF\xBF" + "b"); + REQUIRE(o->string() != + "a\xF0\x8F\xBF\xBF" + "b"); + + // The boundary below it, and a genuine supplementary code point built + // from a surrogate pair, both keep their existing encodings. + json_pull_ptr jp2 = json_begin_string("\"\\uFFFE\""); + json_object_ptr o2 = json_read_tree(jp2); + REQUIRE(o2 != nullptr); + REQUIRE(o2->string() == "\xEF\xBF\xBE"); + + json_pull_ptr jp3 = json_begin_string("\"\\uD83D\\uDC00\""); + json_object_ptr o3 = json_read_tree(jp3); + REQUIRE(o3 != nullptr); + REQUIRE(o3->string() == "\xF0\x9F\x90\x80"); +} + TEST_CASE("Polygon cleaning drops a hole that no ring can parent", "[wagyu]") { // Two mutually reversed self-intersecting rings whose union leaves a hole // that wagyu's topology correction cannot assign to any surviving parent @@ -145,8 +504,8 @@ TEST_CASE("Polygon cleaning drops a hole that no ring can parent", "[wagyu]") { // through the "Could not properly place hole to a parent." handler in // clean_or_clip_poly instead of returning. static const std::vector>> rings = { - {{0, 5}, {5, 4}, {5, 1}, {4, 4}, {4, 2}, {7, 1}, {0, 5}}, - {{0, 5}, {7, 1}, {4, 2}, {4, 4}, {5, 1}, {5, 4}, {0, 0}, {0, 5}}, + {{0, 5}, {5, 4}, {5, 1}, {4, 4}, {4, 2}, {7, 1}, {0, 5}}, + {{0, 5}, {7, 1}, {4, 2}, {4, 4}, {5, 1}, {5, 4}, {0, 0}, {0, 5}}, }; drawvec geom; diff --git a/version.hpp b/version.hpp index 71b353ad..bc3b429b 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v2.81.0" +#define VERSION "v2.82.0" #endif