Compare commits

..
2 Commits
27 changed files with 1071 additions and 1498 deletions
-34
View File
@@ -1,37 +1,3 @@
# 2.82.0
* Fix corruption of a JSON array when a non-final element was removed from it.
The old `json_free` / `json_disconnect` passed an element count to `memmove`
where a byte count was required, so pruning element 0 of an 8-element array
left the first two slots pointing at the same node -- a double free at
teardown -- and silently dropped the last element. Only reachable through a
whole-document tree, because removing the most recently added element made
the bad `memmove` a zero-length no-op. (#388)
* Rewrite `jsonpull` in C++ with `unique_ptr` ownership, `std::vector` for
arrays and hash entries, and `std::string` for string values, replacing the
hand-rolled `malloc`/`realloc`/`free` memory management. Value payloads now
live in type-tagged subclasses reached through asserting accessors, so
reading a hash as a string fails immediately instead of silently returning
garbage. (#388)
* Fix `tippecanoe-json-tool --extract` crashing on a numeric attribute, which
read the number's storage as a string pointer. (#388)
* Fix decoding of a `\u` escape sequence in which a high surrogate is followed
by a non-surrogate BMP code point, which combined the two into a single
wrong code point. (#388)
* Fix a `\uFFFF` escape being decoded to the overlong, invalid four-byte
UTF-8 sequence `F0 8F BF BF` instead of `EF BF BF`. (#388)
* Fix tile-join reading a non-string field type out of a tileset's tilejson.
(#388)
* Fix `tippecanoe-decode` and tile-join crashing on a directory tileset whose
`metadata.json` holds a non-string value, such as a numeric `minzoom` or a
nested object. Those entries are now reported and skipped. (#388)
* Fix an uninitialized read when a prefilter or postfilter emitted a feature
with `"properties": null`. Both filter readers accepted a null `properties`
and then read its length as though it were a hash, which was never
initialized for a non-container node. (#388)
* Fix the include guard in `evaluator.hpp`, which defined `EVALUATOR HPP`
instead of `EVALUATOR_HPP` and so never guarded anything. (#388)
# 2.81.0
* Add `--drop-by-attribute-as-needed=`*attribute* to drop the features with
+1 -9
View File
@@ -128,7 +128,7 @@ clean:
rm -f ./tippecanoe ./tippecanoe-* ./tile-join ./unit *.o *.d */*.o */*.d tests/**/*.mbtiles tests/**/*.check
indent:
clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false, SortIncludes: false}" $(filter-out flatgeobuf.cpp,$(C)) $(H) jsonpull/jsonpull.h jsonpull/jsonpull.cpp
clang-format -i -style="{BasedOnStyle: Google, IndentWidth: 8, UseTab: Always, AllowShortIfStatementsOnASingleLine: false, ColumnLimit: 0, ContinuationIndentWidth: 8, SpaceAfterCStyleCast: true, IndentCaseLabels: false, AllowShortBlocksOnASingleLine: false, AllowShortFunctionsOnASingleLine: false, SortIncludes: false}" $(filter-out flatgeobuf.cpp,$(C)) $(H) jsonpull/*.[ch]
TESTS = $(wildcard tests/*/out/*.json)
SPACE = $(NULL) $(NULL)
@@ -261,14 +261,6 @@ raw-tiles-test: tippecanoe tippecanoe-decode tile-join
./tippecanoe-decode -x generator tests/raw-tiles/nothing > tests/raw-tiles/nothing.json.check
cmp tests/raw-tiles/nothing.json.check tests/raw-tiles/nothing.json
rm -r tests/raw-tiles/nothing tests/raw-tiles/nothing.json.check
# Test that a non-string value in metadata.json is reported and skipped
# instead of being read as a string (which used to crash)
./tippecanoe -q -f -e tests/raw-tiles/nonstring tests/raw-tiles/hackspots.geojson
sed -i.bak 's/"minzoom": "0"/"minzoom": 0/' tests/raw-tiles/nonstring/metadata.json
rm tests/raw-tiles/nonstring/metadata.json.bak
grep -q '"minzoom": 0' tests/raw-tiles/nonstring/metadata.json
./tippecanoe-decode -x generator tests/raw-tiles/nonstring > /dev/null
rm -r tests/raw-tiles/nonstring
pmtiles-test: tippecanoe tippecanoe-decode tile-join
./tippecanoe -q -f -o tests/pmtiles/hackspots.pmtiles -r1 -pC tests/raw-tiles/hackspots.geojson
+11 -8
View File
@@ -42,10 +42,10 @@ void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribut
void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribute_accum, const char *arg, char **argv) {
if (*arg == '{') {
json_pull_ptr jp = json_begin_string(arg);
json_object_ptr o = json_read_tree(jp);
json_pull *jp = json_begin_string(arg);
json_object *o = json_read_tree(jp);
if (o == nullptr) {
if (o == NULL) {
fprintf(stderr, "%s: -E%s: %s\n", *argv, arg, jp->error);
exit(EXIT_JSON);
}
@@ -55,21 +55,24 @@ void set_attribute_accum(std::unordered_map<std::string, attribute_op> &attribut
exit(EXIT_JSON);
}
for (size_t i = 0; i < o->entries().size(); i++) {
const auto &e = o->entries()[i];
for (size_t i = 0; i < o->value.object.length; i++) {
json_object *k = o->value.object.keys[i];
json_object *v = o->value.object.values[i];
if (e.key->type != JSON_STRING) {
if (k->type != JSON_STRING) {
fprintf(stderr, "%s: -E%s: key %zu not a string\n", *argv, arg, i);
exit(EXIT_JSON);
}
if (e.value->type != JSON_STRING) {
if (v->type != JSON_STRING) {
fprintf(stderr, "%s: -E%s: value %zu not a string\n", *argv, arg, i);
exit(EXIT_JSON);
}
set_attribute_accum(attribute_accum, e.key->string(), e.value->string());
set_attribute_accum(attribute_accum, k->value.string.string, v->value.string.string);
}
json_free(o);
json_end(jp);
return;
}
+8 -11
View File
@@ -248,9 +248,9 @@ sqlite3 *dirmeta2tmp(const char *fname) {
if (f == NULL) {
perror(name.c_str());
} else {
json_pull_ptr jp = json_begin_file(f);
json_object_ptr o = json_read_tree(jp);
if (o == nullptr) {
json_pull *jp = json_begin_file(f);
json_object *o = json_read_tree(jp);
if (o == NULL) {
fprintf(stderr, "%s: metadata parsing error: %s\n", name.c_str(), jp->error);
exit(EXIT_JSON);
}
@@ -260,22 +260,19 @@ sqlite3 *dirmeta2tmp(const char *fname) {
exit(EXIT_JSON);
}
for (const auto &e : o->entries()) {
// Skip, not just warn: a metadata.json from another tool may
// have a numeric minzoom or a nested object, and string()
// asserts on the type.
if (e.key->type != JSON_STRING || e.value->type != JSON_STRING) {
for (size_t i = 0; i < o->value.object.length; i++) {
if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) {
fprintf(stderr, "%s: non-string in metadata\n", name.c_str());
continue;
}
char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", e.key->string().c_str(), e.value->string().c_str());
char *sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", o->value.object.keys[i]->value.string.string, o->value.object.values[i]->value.string.string);
if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set %s in metadata: %s\n", e.key->string().c_str(), err);
fprintf(stderr, "set %s in metadata: %s\n", o->value.object.keys[i]->value.string.string, err);
}
sqlite3_free(sql);
}
json_end(jp);
fclose(f);
}
+101 -91
View File
@@ -17,7 +17,7 @@ int compare(mvt_value const &one, json_object *two, bool &fail) {
return false; // string vs non-string
}
return strcmp(one.c_str(), two->string().c_str());
return strcmp(one.c_str(), two->value.string.string);
case mvt_double:
case mvt_float:
@@ -52,9 +52,9 @@ int compare(mvt_value const &one, json_object *two, bool &fail) {
exit(EXIT_IMPOSSIBLE);
}
if (v < two->number()) {
if (v < two->value.number.number) {
return -1;
} else if (v > two->number()) {
} else if (v > two->value.number.number) {
return 1;
} else {
return 0;
@@ -92,7 +92,7 @@ int compare(mvt_value const &one, json_object *two, bool &fail) {
// 1: true
// -1: incomparable (sql null), treated as false in final output
static int eval(std::function<mvt_value(std::string const &)> feature, json_object *f, std::set<std::string> &exclude_attributes, std::vector<std::string> const &unidecode_data) {
if (f != nullptr) {
if (f != NULL) {
if (f->type == JSON_TRUE) {
return 1;
} else if (f->type == JSON_FALSE) {
@@ -102,7 +102,7 @@ static int eval(std::function<mvt_value(std::string const &)> feature, json_obje
}
if (f->type == JSON_NUMBER) {
if (f->number() == 0) {
if (f->value.number.number == 0) {
return 0;
} else {
return 1;
@@ -110,7 +110,7 @@ static int eval(std::function<mvt_value(std::string const &)> feature, json_obje
}
if (f->type == JSON_STRING) {
if (f->string().empty()) {
if (f->value.string.string[0] == '\0') {
return 0;
} else {
return 1;
@@ -118,129 +118,131 @@ static int eval(std::function<mvt_value(std::string const &)> feature, json_obje
}
}
if (f == nullptr || f->type != JSON_ARRAY) {
fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f).c_str());
if (f == NULL || f->type != JSON_ARRAY) {
fprintf(stderr, "Filter is not an array: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
if (f->array().size() < 1) {
fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str());
if (f->value.array.length < 1) {
fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
if (f->array()[0]->type != JSON_STRING) {
fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f).c_str());
if (f->value.array.array[0]->type != JSON_STRING) {
fprintf(stderr, "Filter operation is not a string: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
const std::string &op = f->array()[0]->string();
if (op == "has" ||
op == "!has") {
if (f->array().size() != 2) {
fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str());
if (strcmp(f->value.array.array[0]->value.string.string, "has") == 0 ||
strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) {
if (f->value.array.length != 2) {
fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
if (op == "has") {
if (f->array()[1]->type != JSON_STRING) {
fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f).c_str());
if (strcmp(f->value.array.array[0]->value.string.string, "has") == 0) {
if (f->value.array.array[1]->type != JSON_STRING) {
fprintf(stderr, "\"has\" key is not a string: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
return feature(f->array()[1]->string()).type != mvt_no_such_key;
return feature(std::string(f->value.array.array[1]->value.string.string)).type != mvt_no_such_key;
}
if (op == "!has") {
if (f->array()[1]->type != JSON_STRING) {
fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f).c_str());
if (strcmp(f->value.array.array[0]->value.string.string, "!has") == 0) {
if (f->value.array.array[1]->type != JSON_STRING) {
fprintf(stderr, "\"!has\" key is not a string: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
return feature(f->array()[1]->string()).type == mvt_no_such_key;
return feature(std::string(f->value.array.array[1]->value.string.string)).type == mvt_no_such_key;
}
}
if (op == "==" ||
op == "!=" ||
op == ">" ||
op == ">=" ||
op == "<" ||
op == "<=") {
if (f->array().size() != 3) {
fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str());
if (strcmp(f->value.array.array[0]->value.string.string, "==") == 0 ||
strcmp(f->value.array.array[0]->value.string.string, "!=") == 0 ||
strcmp(f->value.array.array[0]->value.string.string, ">") == 0 ||
strcmp(f->value.array.array[0]->value.string.string, ">=") == 0 ||
strcmp(f->value.array.array[0]->value.string.string, "<") == 0 ||
strcmp(f->value.array.array[0]->value.string.string, "<=") == 0) {
if (f->value.array.length != 3) {
fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
if (f->array()[1]->type != JSON_STRING) {
fprintf(stderr, "comparison key is not a string: %s\n", json_stringify(f).c_str());
if (f->value.array.array[1]->type != JSON_STRING) {
fprintf(stderr, "comparison key is not a string: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
mvt_value ff = feature(f->array()[1]->string());
mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string));
if (ff.type == mvt_no_such_key) {
static bool warned = false;
if (!warned) {
fprintf(stderr, "Warning: attribute not found for comparison: %s\n", json_stringify(f).c_str());
const char *s = json_stringify(f);
fprintf(stderr, "Warning: attribute not found for comparison: %s\n", s);
free((void *) s);
warned = true;
}
if (op == "!=") {
if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) {
return true; // attributes that aren't found are not equal
}
return false; // not found: comparison is false
}
bool fail = false;
int cmp = compare(ff, f->array()[2].get(), fail);
int cmp = compare(ff, f->value.array.array[2], fail);
if (fail) {
static bool warned = false;
if (!warned) {
fprintf(stderr, "Warning: mismatched type in comparison: %s\n", json_stringify(f).c_str());
const char *s = json_stringify(f);
fprintf(stderr, "Warning: mismatched type in comparison: %s\n", s);
free((void *) s);
warned = true;
}
if (op == "!=") {
if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) {
return true; // mismatched types are not equal
}
return false;
}
if (op == "==") {
if (strcmp(f->value.array.array[0]->value.string.string, "==") == 0) {
return cmp == 0;
}
if (op == "!=") {
if (strcmp(f->value.array.array[0]->value.string.string, "!=") == 0) {
return cmp != 0;
}
if (op == ">") {
if (strcmp(f->value.array.array[0]->value.string.string, ">") == 0) {
return cmp > 0;
}
if (op == ">=") {
if (strcmp(f->value.array.array[0]->value.string.string, ">=") == 0) {
return cmp >= 0;
}
if (op == "<") {
if (strcmp(f->value.array.array[0]->value.string.string, "<") == 0) {
return cmp < 0;
}
if (op == "<=") {
if (strcmp(f->value.array.array[0]->value.string.string, "<=") == 0) {
return cmp <= 0;
}
fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f).c_str());
fprintf(stderr, "Internal error: can't happen: %s\n", json_stringify(f));
exit(EXIT_IMPOSSIBLE);
}
if (op == "all" ||
op == "any" ||
op == "none") {
if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0 ||
strcmp(f->value.array.array[0]->value.string.string, "any") == 0 ||
strcmp(f->value.array.array[0]->value.string.string, "none") == 0) {
bool v;
if (op == "all") {
if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) {
v = true;
} else {
v = false;
}
for (size_t i = 1; i < f->array().size(); i++) {
int out = eval(feature, f->array()[i].get(), exclude_attributes, unidecode_data);
for (size_t i = 1; i < f->value.array.length; i++) {
int out = eval(feature, f->value.array.array[i], exclude_attributes, unidecode_data);
if (out >= 0) { // nulls are ignored in boolean and/or expressions
if (op == "all") {
if (strcmp(f->value.array.array[0]->value.string.string, "all") == 0) {
v = v && out;
if (!v) {
break;
@@ -254,47 +256,51 @@ static int eval(std::function<mvt_value(std::string const &)> feature, json_obje
}
}
if (op == "none") {
if (strcmp(f->value.array.array[0]->value.string.string, "none") == 0) {
return !v;
} else {
return v;
}
}
if (op == "in" ||
op == "!in") {
if (f->array().size() < 2) {
fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f).c_str());
if (strcmp(f->value.array.array[0]->value.string.string, "in") == 0 ||
strcmp(f->value.array.array[0]->value.string.string, "!in") == 0) {
if (f->value.array.length < 2) {
fprintf(stderr, "Array too small in filter: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
if (f->array()[1]->type != JSON_STRING) {
fprintf(stderr, "\"!in\" key is not a string: %s\n", json_stringify(f).c_str());
if (f->value.array.array[1]->type != JSON_STRING) {
fprintf(stderr, "\"!in\" key is not a string: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
mvt_value ff = feature(f->array()[1]->string());
mvt_value ff = feature(std::string(f->value.array.array[1]->value.string.string));
if (ff.type == mvt_no_such_key) {
static bool warned = false;
if (!warned) {
fprintf(stderr, "Warning: attribute not found for comparison: %s\n", json_stringify(f).c_str());
const char *s = json_stringify(f);
fprintf(stderr, "Warning: attribute not found for comparison: %s\n", s);
free((void *) s);
warned = true;
}
if (op == "!in") {
if (strcmp(f->value.array.array[0]->value.string.string, "!in") == 0) {
return true; // attributes that aren't found are not in
}
return false; // not found: comparison is false
}
bool found = false;
for (size_t i = 2; i < f->array().size(); i++) {
for (size_t i = 2; i < f->value.array.length; i++) {
bool fail = false;
int cmp = compare(ff, f->array()[i].get(), fail);
int cmp = compare(ff, f->value.array.array[i], fail);
if (fail) {
static bool warned = false;
if (!warned) {
fprintf(stderr, "Warning: mismatched type in comparison: %s\n", json_stringify(f).c_str());
const char *s = json_stringify(f);
fprintf(stderr, "Warning: mismatched type in comparison: %s\n", s);
free((void *) s);
warned = true;
}
cmp = 1;
@@ -306,39 +312,39 @@ static int eval(std::function<mvt_value(std::string const &)> feature, json_obje
}
}
if (op == "in") {
if (strcmp(f->value.array.array[0]->value.string.string, "in") == 0) {
return found;
} else {
return !found;
}
}
if (op == "attribute-filter") {
if (f->array().size() != 3) {
fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f).c_str());
if (strcmp(f->value.array.array[0]->value.string.string, "attribute-filter") == 0) {
if (f->value.array.length != 3) {
fprintf(stderr, "Wrong number of array elements in filter: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
if (f->array()[1]->type != JSON_STRING) {
fprintf(stderr, "\"attribute-filter\" key is not a string: %s\n", json_stringify(f).c_str());
if (f->value.array.array[1]->type != JSON_STRING) {
fprintf(stderr, "\"attribute-filter\" key is not a string: %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
bool ok = eval(feature, f->array()[2].get(), exclude_attributes, unidecode_data) > 0;
bool ok = eval(feature, f->value.array.array[2], exclude_attributes, unidecode_data) > 0;
if (!ok) {
exclude_attributes.insert(f->array()[1]->string());
exclude_attributes.insert(f->value.array.array[1]->value.string.string);
}
return true;
}
fprintf(stderr, "Unknown filter %s\n", json_stringify(f).c_str());
fprintf(stderr, "Unknown filter %s\n", json_stringify(f));
exit(EXIT_FILTER);
}
static bool evaluate(std::function<mvt_value(std::string const &)> feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes, std::vector<std::string> const &unidecode_data) {
if (filter == nullptr || filter->type != JSON_HASH) {
fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter).c_str());
bool evaluate(std::function<mvt_value(std::string const &)> feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes, std::vector<std::string> const &unidecode_data) {
if (filter == NULL || filter->type != JSON_HASH) {
fprintf(stderr, "Error: filter is not a hash: %s\n", json_stringify(filter));
exit(EXIT_JSON);
}
@@ -346,43 +352,47 @@ static bool evaluate(std::function<mvt_value(std::string const &)> feature, std:
json_object *f;
f = json_hash_get(filter, layer.c_str());
if (ok && f != nullptr) {
if (ok && f != NULL) {
ok = eval(feature, f, exclude_attributes, unidecode_data) > 0;
}
f = json_hash_get(filter, "*");
if (ok && f != nullptr) {
if (ok && f != NULL) {
ok = eval(feature, f, exclude_attributes, unidecode_data) > 0;
}
return ok;
}
json_object_ptr read_filter(const char *fname) {
json_object *read_filter(const char *fname) {
FILE *fp = fopen(fname, "r");
if (fp == NULL) {
perror(fname);
exit(EXIT_OPEN);
}
json_pull_ptr jp = json_begin_file(fp);
json_object_ptr filter = json_read_tree(jp);
if (filter == nullptr) {
json_pull *jp = json_begin_file(fp);
json_object *filter = json_read_tree(jp);
if (filter == NULL) {
fprintf(stderr, "%s: %s\n", fname, jp->error);
exit(EXIT_JSON);
}
json_disconnect(filter);
json_end(jp);
fclose(fp);
return filter;
}
json_object_ptr parse_filter(const char *s) {
json_pull_ptr jp = json_begin_string(s);
json_object_ptr filter = json_read_tree(jp);
if (filter == nullptr) {
json_object *parse_filter(const char *s) {
json_pull *jp = json_begin_string(s);
json_object *filter = json_read_tree(jp);
if (filter == NULL) {
fprintf(stderr, "Could not parse filter %s\n", s);
fprintf(stderr, "%s\n", jp->error);
exit(EXIT_JSON);
}
json_disconnect(filter);
json_end(jp);
return filter;
}
+3 -7
View File
@@ -1,5 +1,5 @@
#ifndef EVALUATOR_HPP
#define EVALUATOR_HPP
#define EVALUATOR HPP
#include <unordered_map>
#include <string>
@@ -7,13 +7,9 @@
#include "jsonpull/jsonpull.h"
#include "mvt.hpp"
// The `filter` parameters take a borrowed pointer; the caller (in
// main.cpp, tile-join, overzoom) keeps the json_object_ptr alive
// across every per-feature evaluate() call. A raw pointer avoids
// touching unique_ptr at all on this hot path.
bool evaluate(std::unordered_map<std::string, mvt_value> const &feature, std::string const &layer, json_object *filter, std::set<std::string> &exclude_attributes, std::vector<std::string> const &unidecode_data);
json_object_ptr parse_filter(const char *s);
json_object_ptr read_filter(const char *fname);
json_object *parse_filter(const char *s);
json_object *read_filter(const char *fname);
bool evaluate(mvt_feature const &feat, mvt_layer const &layer, json_object *filter, std::set<std::string> &exclude_attributes, int z, std::vector<std::string> const &unidecode_data);
+12 -9
View File
@@ -394,25 +394,28 @@ void readFeature(protozero::pbf_reader &pbf, size_t dim, double e, std::vector<s
auto tip = other.find("tippecanoe");
if (tip != other.end()) {
json_pull_ptr jp = json_begin_string(tip->second.s.c_str());
json_object_ptr o = json_read_tree(jp);
json_pull *jp = json_begin_string(tip->second.s.c_str());
json_object *o = json_read_tree(jp);
if (o != nullptr) {
if (o != NULL) {
json_object *min = json_hash_get(o, "minzoom");
if (min != nullptr && (min->type == JSON_NUMBER)) {
sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number()));
if (min != NULL && (min->type == JSON_NUMBER)) {
sf.tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number));
}
json_object *max = json_hash_get(o, "maxzoom");
if (max != nullptr && (max->type == JSON_NUMBER)) {
sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number()));
if (max != NULL && (max->type == JSON_NUMBER)) {
sf.tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number));
}
json_object *tlayer = json_hash_get(o, "layer");
if (tlayer != nullptr && (tlayer->type == JSON_STRING)) {
layername = tlayer->string();
if (tlayer != NULL && (tlayer->type == JSON_STRING)) {
layername = tlayer->value.string.string;
}
}
json_free(o);
json_end(jp);
}
serialize_feature(sst, sf, layername);
+27 -28
View File
@@ -12,7 +12,6 @@
#include <errno.h>
#include "geojson-loop.hpp"
#include "jsonpull/jsonpull.h"
#include "text.hpp"
// XXX duplicated
#define GEOM_TYPES 6
@@ -27,34 +26,34 @@ static const char *geometry_names[GEOM_TYPES] = {
// XXX duplicated
static void json_context(json_object *j) {
std::string s = json_stringify(j);
char *s = json_stringify(j);
if (s.size() >= 500) {
s = truncate_string(s, 497);
s.append("...");
if (strlen(s) >= 500) {
snprintf(s + 497, strlen(s) + 1 - 497, "...");
}
fprintf(stderr, "in JSON object %s\n", s.c_str());
fprintf(stderr, "in JSON object %s\n", s);
free(s); // stringify
}
void parse_json(json_feature_action *jfa, json_pull_ptr &jp) {
void parse_json(json_feature_action *jfa, json_pull *jp) {
long long found_hashes = 0;
long long found_features = 0;
long long found_geometries = 0;
while (1) {
json_object *j = json_read(jp);
if (j == nullptr) {
if (jp->error != nullptr) {
if (j == NULL) {
if (jp->error != NULL) {
fprintf(stderr, "%s:%d: %s: ", jfa->fname.c_str(), jp->line, jp->error);
if (jp->root != nullptr) {
json_context(jp->root.get());
if (jp->root != NULL) {
json_context(jp->root);
} else {
fprintf(stderr, "\n");
}
}
jp->root.reset();
json_free(jp->root);
break;
}
@@ -67,7 +66,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr &jp) {
}
json_object *type = json_hash_get(j, "type");
if (type == nullptr || type->type != JSON_STRING) {
if (type == NULL || type->type != JSON_STRING) {
continue;
}
@@ -75,25 +74,25 @@ void parse_json(json_feature_action *jfa, json_pull_ptr &jp) {
int i;
int is_geometry = 0;
for (i = 0; i < GEOM_TYPES; i++) {
if (type->string() == geometry_names[i]) {
if (strcmp(type->value.string.string, geometry_names[i]) == 0) {
is_geometry = 1;
break;
}
}
if (is_geometry) {
if (j->parent != nullptr) {
if (j->parent->type == JSON_ARRAY && j->parent->parent != nullptr) {
if (j->parent != NULL) {
if (j->parent->type == JSON_ARRAY && j->parent->parent != NULL) {
if (j->parent->parent->type == JSON_HASH) {
json_object *geometries = json_hash_get(j->parent->parent, "geometries");
if (geometries != nullptr) {
if (geometries != NULL) {
// Parent of Parent must be a GeometryCollection
is_geometry = 0;
}
}
} else if (j->parent->type == JSON_HASH) {
json_object *geometry = json_hash_get(j->parent, "geometry");
if (geometry != nullptr) {
if (geometry != NULL) {
// Parent must be a Feature
is_geometry = 0;
}
@@ -103,8 +102,8 @@ void parse_json(json_feature_action *jfa, json_pull_ptr &jp) {
if (is_geometry) {
json_object *jo = j;
while (jo != nullptr) {
if (jo->parent != nullptr && jo->parent->type == JSON_HASH) {
while (jo != NULL) {
if (jo->parent != NULL && jo->parent->type == JSON_HASH) {
if (json_hash_get(jo->parent, "properties") == jo) {
// Ancestor is the value corresponding to a properties key
is_geometry = 0;
@@ -121,14 +120,14 @@ void parse_json(json_feature_action *jfa, json_pull_ptr &jp) {
}
found_geometries++;
jfa->add_feature(j, false, nullptr, nullptr, nullptr, j);
jfa->add_feature(j, false, NULL, NULL, NULL, j);
json_free(j);
continue;
}
}
if (type->string() != "Feature") {
if (type->string() == "FeatureCollection") {
if (strcmp(type->value.string.string, "Feature") != 0) {
if (strcmp(type->value.string.string, "FeatureCollection") == 0) {
jfa->check_crs(j);
json_free(j);
}
@@ -142,7 +141,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr &jp) {
found_features++;
json_object *geometry = json_hash_get(j, "geometry");
if (geometry == nullptr) {
if (geometry == NULL) {
fprintf(stderr, "%s:%d: feature with no geometry: ", jfa->fname.c_str(), jp->line);
json_context(j);
json_free(j);
@@ -150,7 +149,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr &jp) {
}
json_object *properties = json_hash_get(j, "properties");
if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
fprintf(stderr, "%s:%d: feature without properties hash: ", jfa->fname.c_str(), jp->line);
json_context(j);
json_free(j);
@@ -160,8 +159,8 @@ void parse_json(json_feature_action *jfa, json_pull_ptr &jp) {
bool is_feature = true;
{
json_object *jo = j;
while (jo != nullptr) {
if (jo->parent != nullptr && jo->parent->type == JSON_HASH) {
while (jo != NULL) {
if (jo->parent != NULL && jo->parent->type == JSON_HASH) {
if (json_hash_get(jo->parent, "properties") == jo) {
// Ancestor is the value corresponding to a properties key
is_feature = false;
@@ -179,7 +178,7 @@ void parse_json(json_feature_action *jfa, json_pull_ptr &jp) {
json_object *id = json_hash_get(j, "id");
json_object *geometries = json_hash_get(geometry, "geometries");
if (geometries != nullptr && geometries->type == JSON_ARRAY) {
if (geometries != NULL && geometries->type == JSON_ARRAY) {
jfa->add_feature(geometries, true, properties, id, tippecanoe, j);
} else {
jfa->add_feature(geometry, false, properties, id, tippecanoe, j);
+1 -1
View File
@@ -8,4 +8,4 @@ struct json_feature_action {
virtual void check_crs(json_object *j) = 0;
};
void parse_json(json_feature_action *action, json_pull_ptr &jp);
void parse_json(json_feature_action *action, json_pull *jp);
+50 -52
View File
@@ -42,7 +42,7 @@
int serialize_geojson_feature(struct serialization_state *sst, json_object *geometry, json_object *properties, json_object *id, int layer, json_object *tippecanoe, json_object *feature, std::string const &layername) {
json_object *geometry_type = json_hash_get(geometry, "type");
if (geometry_type == nullptr) {
if (geometry_type == NULL) {
static int warned = 0;
if (!warned) {
fprintf(stderr, "%s:%d: null geometry (additional not reported): ", sst->fname, sst->line);
@@ -60,7 +60,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
}
json_object *coordinates = json_hash_get(geometry, "coordinates");
if (coordinates == nullptr || coordinates->type != JSON_ARRAY) {
if (coordinates == NULL || coordinates->type != JSON_ARRAY) {
fprintf(stderr, "%s:%d: feature without coordinates array: ", sst->fname, sst->line);
json_context(feature);
return 0;
@@ -68,12 +68,12 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
int t;
for (t = 0; t < GEOM_TYPES; t++) {
if (geometry_type->string() == geometry_names[t]) {
if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) {
break;
}
}
if (t >= GEOM_TYPES) {
fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->string().c_str());
fprintf(stderr, "%s:%d: Can't handle geometry type %s: ", sst->fname, sst->line, geometry_type->value.string.string);
json_context(feature);
return 0;
}
@@ -82,48 +82,48 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
int tippecanoe_maxzoom = -1;
std::string tippecanoe_layername = layername;
if (tippecanoe != nullptr) {
if (tippecanoe != NULL) {
json_object *min = json_hash_get(tippecanoe, "minzoom");
if (min != nullptr && (min->type == JSON_NUMBER)) {
tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->number()));
if (min != NULL && (min->type == JSON_NUMBER)) {
tippecanoe_minzoom = integer_zoom(sst->fname, milo::dtoa_milo(min->value.number.number));
}
json_object *max = json_hash_get(tippecanoe, "maxzoom");
if (max != nullptr && (max->type == JSON_NUMBER)) {
tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->number()));
if (max != NULL && (max->type == JSON_NUMBER)) {
tippecanoe_maxzoom = integer_zoom(sst->fname, milo::dtoa_milo(max->value.number.number));
}
json_object *ln = json_hash_get(tippecanoe, "layer");
if (ln != nullptr && (ln->type == JSON_STRING)) {
tippecanoe_layername = ln->string();
if (ln != NULL && (ln->type == JSON_STRING)) {
tippecanoe_layername = std::string(ln->value.string.string);
}
}
bool has_id = false;
unsigned long long id_value = 0;
if (id != nullptr) {
if (id != NULL) {
if (id->type == JSON_NUMBER) {
if (id->number() >= 0) {
if (id->value.number.number >= 0) {
char *err = NULL;
std::string id_number = milo::dtoa_milo(id->number());
std::string id_number = milo::dtoa_milo(id->value.number.number);
id_value = strtoull(id_number.c_str(), &err, 10);
if (id->large_unsigned() != 0) {
id_value = id->large_unsigned();
if (id->value.number.large_unsigned != 0) {
id_value = id->value.number.large_unsigned;
}
if (err != NULL && *err != '\0') {
static bool warned_frac = false;
if (!warned_frac) {
fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", milo::dtoa_milo(id->number()).c_str());
fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str());
warned_frac = true;
}
} else if (id->large_unsigned() == 0 && std::to_string(id_value) != milo::dtoa_milo(id->number())) {
} else if (id->value.number.large_unsigned == 0 && std::to_string(id_value) != milo::dtoa_milo(id->value.number.number)) {
static bool warned = false;
if (!warned) {
fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", milo::dtoa_milo(id->number()).c_str());
fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str());
warned = true;
}
} else {
@@ -133,7 +133,7 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
static bool warned_neg = false;
if (!warned_neg) {
fprintf(stderr, "Warning: Can't represent negative feature ID %s\n", milo::dtoa_milo(id->number()).c_str());
fprintf(stderr, "Warning: Can't represent negative feature ID %s\n", milo::dtoa_milo(id->value.number.number).c_str());
warned_neg = true;
}
}
@@ -142,20 +142,20 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
if (additional[A_CONVERT_NUMERIC_IDS] && id->type == JSON_STRING) {
char *err = NULL;
id_value = strtoull(id->string().c_str(), &err, 10);
id_value = strtoull(id->value.string.string, &err, 10);
if (err != NULL && *err != '\0') {
static bool warned_frac = false;
if (!warned_frac) {
fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->string().c_str());
fprintf(stderr, "Warning: Can't represent non-integer feature ID %s\n", id->value.string.string);
warned_frac = true;
}
} else if (std::to_string(id_value) != id->string()) {
} else if (std::to_string(id_value) != id->value.string.string) {
static bool warned = false;
if (!warned) {
fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->string().c_str());
fprintf(stderr, "Warning: Can't represent too-large feature ID %s\n", id->value.string.string);
warned = true;
}
} else {
@@ -168,29 +168,33 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
static bool warned_nan = false;
if (!warned_nan) {
fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", json_stringify(id).c_str());
char *s = json_stringify(id);
fprintf(stderr, "Warning: Can't represent non-numeric feature ID %s\n", s);
free(s); // stringify
warned_nan = true;
}
}
}
}
size_t nprop = 0;
if (properties != NULL && properties->type == JSON_HASH) {
nprop = properties->value.object.length;
}
std::vector<std::shared_ptr<std::string>> full_keys;
std::vector<serial_val> values;
full_keys.reserve(nprop);
values.reserve(nprop);
key_pool key_pool;
if (properties != nullptr && properties->type == JSON_HASH) {
const auto &entries = properties->entries();
full_keys.reserve(entries.size());
values.reserve(entries.size());
for (size_t i = 0; i < nprop; i++) {
if (properties->value.object.keys[i]->type == JSON_STRING) {
serial_val sv = stringify_value(properties->value.object.values[i], sst->fname, sst->line, feature);
for (const auto &e : entries) {
if (e.key->type == JSON_STRING) {
serial_val sv = stringify_value(e.value.get(), sst->fname, sst->line, feature);
full_keys.emplace_back(key_pool.pool(e.key->string()));
values.push_back(std::move(sv));
}
full_keys.emplace_back(key_pool.pool(properties->value.object.keys[i]->value.string.string));
values.push_back(std::move(sv));
}
}
@@ -216,14 +220,14 @@ int serialize_geojson_feature(struct serialization_state *sst, json_object *geom
void check_crs(json_object *j, const char *reading) {
json_object *crs = json_hash_get(j, "crs");
if (crs != nullptr) {
if (crs != NULL) {
json_object *properties = json_hash_get(crs, "properties");
if (properties != nullptr) {
if (properties != NULL) {
json_object *name = json_hash_get(properties, "name");
if (name != nullptr && name->type == JSON_STRING) {
if (name->string() != projection->alias) {
if (name != NULL && name->type == JSON_STRING) {
if (strcmp(name->value.string.string, projection->alias) != 0) {
if (!quiet) {
fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->string().c_str(), projection->alias);
fprintf(stderr, "%s: Warning: GeoJSON specified projection \"%s\", not the expected \"%s\".\n", reading, name->value.string.string, projection->alias);
fprintf(stderr, "%s: If \"%s\" is not the expected projection, use -s to specify the right one.\n", reading, projection->alias);
}
}
@@ -238,13 +242,11 @@ struct json_serialize_action : json_feature_action {
std::string layername;
int add_feature(json_object *geometry, bool geometrycollection, json_object *properties, json_object *id, json_object *tippecanoe, json_object *feature) {
// Only json_read results reach this; a detached tree has no parser.
assert(geometry->parser != nullptr);
sst->line = geometry->parser->line;
if (geometrycollection) {
int ret = 1;
for (size_t g = 0; g < geometry->array().size(); g++) {
ret &= serialize_geojson_feature(sst, geometry->array()[g].get(), properties, id, layer, tippecanoe, feature, layername);
for (size_t g = 0; g < geometry->value.array.length; g++) {
ret &= serialize_geojson_feature(sst, geometry->value.array.array[g], properties, id, layer, tippecanoe, feature, layername);
}
return ret;
} else {
@@ -257,7 +259,7 @@ struct json_serialize_action : json_feature_action {
}
};
void parse_json(struct serialization_state *sst, json_pull_ptr &jp, int layer, std::string layername) {
void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername) {
json_serialize_action jsa;
jsa.fname = sst->fname;
jsa.sst = sst;
@@ -294,7 +296,7 @@ ssize_t json_map_read(struct json_pull *jp, char *buffer, size_t n) {
return n;
}
json_pull_ptr json_begin_map(char *map, long long len) {
struct json_pull *json_begin_map(char *map, long long len) {
struct jsonmap *jm = new jsonmap;
if (jm == NULL) {
perror("Out of memory");
@@ -308,11 +310,7 @@ json_pull_ptr json_begin_map(char *map, long long len) {
return json_begin(json_map_read, jm);
}
void json_end_map(json_pull_ptr &jp) {
if (jp == nullptr) {
return;
}
void json_end_map(struct json_pull *jp) {
delete (struct jsonmap *) jp->source;
jp->source = nullptr;
json_end(jp);
}
+6 -6
View File
@@ -10,21 +10,21 @@
#include "serial.hpp"
struct parse_json_args {
json_pull_ptr jp;
json_pull *jp;
int layer;
std::string *layername;
struct serialization_state *sst;
parse_json_args(json_pull_ptr jp1, int layer1, std::string *layername1, struct serialization_state *sst1)
: jp(std::move(jp1)), layer(layer1), layername(layername1), sst(sst1) {
parse_json_args(json_pull *jp1, int layer1, std::string *layername1, struct serialization_state *sst1)
: jp(jp1), layer(layer1), layername(layername1), sst(sst1) {
}
};
json_pull_ptr json_begin_map(char *map, long long len);
void json_end_map(json_pull_ptr &jp);
struct json_pull *json_begin_map(char *map, long long len);
void json_end_map(struct json_pull *jp);
void parse_json(struct serialization_state *sst, json_pull_ptr &jp, int layer, std::string layername);
void parse_json(struct serialization_state *sst, json_pull *jp, int layer, std::string layername);
void *run_parse_json(void *v);
#endif
+499 -360
View File
File diff suppressed because it is too large Load Diff
+54 -281
View File
@@ -1,12 +1,9 @@
#ifndef JSONPULL_H
#define JSONPULL_H
#include <stdio.h>
#include <sys/types.h>
#include <cassert>
#include <memory>
#include <string>
#include <vector>
#ifdef __cplusplus
extern "C" {
#endif
typedef enum json_type {
// These types can be returned by json_read()
@@ -28,298 +25,74 @@ typedef enum json_type {
JSON_VALUE,
} json_type;
struct json_object;
struct json_pull;
typedef struct json_object {
struct json_object *parent;
struct json_pull *parser;
// Ownership model
//
// Every node has exactly one owner: its parent (a json_object_ptr in the
// parent's vector or hash entry), the parser (jp->root) for a top-level value,
// or the caller once json_read_tree / json_disconnect hands the tree over.
// json_read and json_hash_get return borrowed pointers, valid while the owning
// container is intact.
//
// `parent` and `parser` are non-owning, so they cannot form a cycle. Detaching
// clears every `parser` in the subtree, since the json_pull may die first, and
// clears `parent` only on the detached root, which pointed out of the subtree.
// Interior `parent` links stay, so a detached tree is still walkable upwards
// and json_free / json_disconnect still work inside it -- both find a node's
// owner through `parent`.
//
// json_object has no vptr; json_object_deleter switches on `type` and
// static_casts so the right subclass destructor runs. Payloads live in those
// subclasses rather than one wide struct, and the accessors assert on `type`
// before downcasting.
//
// json_pull_ptr is a shared_ptr: a parser is created and freed once.
union {
struct {
double number;
unsigned long long large_unsigned;
long long large_signed;
} number;
// Stateless, so json_object_ptr stays one pointer wide. See Ownership model.
struct json_object_deleter {
void operator()(json_object *p) const noexcept;
};
struct {
char *string;
void *refcon; // reference constant for caller's use
} string;
typedef std::unique_ptr<json_object, json_object_deleter> json_object_ptr;
typedef std::shared_ptr<json_pull> json_pull_ptr;
struct {
struct json_object **array;
size_t length;
} array;
// One key/value pair in a JSON_HASH, held in source order.
struct json_entry {
json_object_ptr key;
json_object_ptr value;
};
struct json_object {
json_object *parent = nullptr;
json_pull *parser = nullptr;
json_type type;
json_object(json_type t)
: type(t) {
}
json_object(json_type t, json_object *p, json_pull *pl)
: parent(p), parser(pl), type(t) {
}
inline std::string &string();
inline const std::string &string() const;
// large_unsigned() / large_signed() return 0 when the number is not
// held in that representation, the convention callers already expect.
inline double number() const;
inline unsigned long long large_unsigned() const;
inline long long large_signed() const;
inline void set_number(double d);
inline void set_large_unsigned(unsigned long long u);
inline void set_large_signed(long long s);
inline std::vector<json_object_ptr> &array();
inline const std::vector<json_object_ptr> &array() const;
inline std::vector<json_entry> &entries();
inline const std::vector<json_entry> &entries() const;
};
struct json_number : json_object {
enum repr_t { REPR_DOUBLE,
REPR_LARGE_UNSIGNED,
REPR_LARGE_SIGNED };
repr_t repr = REPR_DOUBLE;
union value_t {
double d;
unsigned long long u;
long long s;
value_t()
: d(0) {
}
struct {
struct json_object **keys;
struct json_object **values;
size_t length;
} object;
} value;
json_number()
: json_object(JSON_NUMBER) {
}
json_number(json_object *p, json_pull *pl)
: json_object(JSON_NUMBER, p, pl) {
}
};
json_type type;
int expect;
} json_object;
struct json_string : json_object {
std::string string_value;
typedef struct json_pull {
char *error;
int line;
json_string()
: json_object(JSON_STRING) {
}
json_string(json_object *p, json_pull *pl)
: json_object(JSON_STRING, p, pl) {
}
};
ssize_t (*read)(struct json_pull *, char *buf, size_t n);
void *source;
char *buffer;
ssize_t buffer_tail;
ssize_t buffer_head;
struct json_array : json_object {
std::vector<json_object_ptr> array_value;
json_object *container;
json_object *root;
// 2 slots: coordinate pairs dominate the parse workload.
json_array()
: json_object(JSON_ARRAY) {
array_value.reserve(2);
}
json_array(json_object *p, json_pull *pl)
: json_object(JSON_ARRAY, p, pl) {
array_value.reserve(2);
}
};
struct string *number_buffer;
} json_pull;
struct json_hash : json_object {
std::vector<json_entry> entries_value;
json_pull *json_begin_file(FILE *f);
json_pull *json_begin_string(const char *s);
// 4 slots: the typical GeoJSON property hash.
json_hash()
: json_object(JSON_HASH) {
entries_value.reserve(4);
}
json_hash(json_object *p, json_pull *pl)
: json_object(JSON_HASH, p, pl) {
entries_value.reserve(4);
}
};
inline std::string &json_object::string() {
assert(type == JSON_STRING);
return static_cast<json_string *>(this)->string_value;
}
inline const std::string &json_object::string() const {
assert(type == JSON_STRING);
return static_cast<const json_string *>(this)->string_value;
}
inline double json_object::number() const {
assert(type == JSON_NUMBER);
auto *n = static_cast<const json_number *>(this);
switch (n->repr) {
case json_number::REPR_LARGE_UNSIGNED:
return static_cast<double>(n->value.u);
case json_number::REPR_LARGE_SIGNED:
return static_cast<double>(n->value.s);
case json_number::REPR_DOUBLE:
default:
return n->value.d;
}
}
inline unsigned long long json_object::large_unsigned() const {
assert(type == JSON_NUMBER);
auto *n = static_cast<const json_number *>(this);
return n->repr == json_number::REPR_LARGE_UNSIGNED ? n->value.u : 0;
}
inline long long json_object::large_signed() const {
assert(type == JSON_NUMBER);
auto *n = static_cast<const json_number *>(this);
return n->repr == json_number::REPR_LARGE_SIGNED ? n->value.s : 0;
}
inline void json_object::set_number(double d) {
assert(type == JSON_NUMBER);
auto *n = static_cast<json_number *>(this);
n->repr = json_number::REPR_DOUBLE;
n->value.d = d;
}
inline void json_object::set_large_unsigned(unsigned long long u) {
assert(type == JSON_NUMBER);
auto *n = static_cast<json_number *>(this);
n->repr = json_number::REPR_LARGE_UNSIGNED;
n->value.u = u;
}
inline void json_object::set_large_signed(long long s) {
assert(type == JSON_NUMBER);
auto *n = static_cast<json_number *>(this);
n->repr = json_number::REPR_LARGE_SIGNED;
n->value.s = s;
}
inline std::vector<json_object_ptr> &json_object::array() {
assert(type == JSON_ARRAY);
return static_cast<json_array *>(this)->array_value;
}
inline const std::vector<json_object_ptr> &json_object::array() const {
assert(type == JSON_ARRAY);
return static_cast<const json_array *>(this)->array_value;
}
inline std::vector<json_entry> &json_object::entries() {
assert(type == JSON_HASH);
return static_cast<json_hash *>(this)->entries_value;
}
inline const std::vector<json_entry> &json_object::entries() const {
assert(type == JSON_HASH);
return static_cast<const json_hash *>(this)->entries_value;
}
inline void json_object_deleter::operator()(json_object *p) const noexcept {
if (p == nullptr) {
return;
}
switch (p->type) {
case JSON_NUMBER:
delete static_cast<json_number *>(p);
break;
case JSON_STRING:
delete static_cast<json_string *>(p);
break;
case JSON_ARRAY:
delete static_cast<json_array *>(p);
break;
case JSON_HASH:
delete static_cast<json_hash *>(p);
break;
default:
// JSON_TRUE / JSON_FALSE / JSON_NULL: no extra fields.
delete p;
break;
}
}
struct json_pull {
const char *error = nullptr; // points at a string literal; no allocation
int line = 1;
ssize_t (*read)(struct json_pull *, char *buf, size_t n) = nullptr;
void *source = nullptr;
std::vector<char> buffer;
ssize_t buffer_tail = 0;
ssize_t buffer_head = 0;
// Currently-open containers, innermost last, each with the token it
// expects next. `container` is borrowed; the owner is the surrounding
// container, or `root` for the outermost.
struct parse_frame {
json_object *container;
json_type expect;
};
std::vector<parse_frame> container_stack;
// Most recently completed top-level value, owned until the next one
// starts parsing or the caller takes it.
json_object_ptr root;
// Reused across tokens, cleared but not shrunk, so they stop
// reallocating once grown to the largest token seen.
std::string number_buffer;
std::string string_buffer;
};
json_pull_ptr json_begin_file(FILE *f);
json_pull_ptr json_begin_string(const char *s);
json_pull_ptr json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source);
// Resets the caller's pointer. Optional: the parser frees itself when the
// last json_pull_ptr to it goes away.
void json_end(json_pull_ptr &p);
json_pull *json_begin(ssize_t (*read)(struct json_pull *, char *buffer, size_t n), void *source);
void json_end(json_pull *p);
typedef void (*json_separator_callback)(json_type type, json_pull *j, void *state);
// The next completed node, borrowed. Valid until the next call that extends
// or trims the parser's tree. nullptr at end of input or on error.
json_object *json_read(json_pull_ptr &j);
json_object *json_read_separators(json_pull_ptr &j, json_separator_callback cb, void *state);
json_object *json_read_tree(json_pull *j);
json_object *json_read(json_pull *j);
json_object *json_read_separators(json_pull *j, json_separator_callback cb, void *state);
void json_free(json_object *j);
void json_disconnect(json_object *j);
// Drains the next top-level value out of the parser and hands it over. The
// returned tree can outlive the json_pull. See Ownership model for what the
// detach does and does not clear.
json_object_ptr json_read_tree(json_pull_ptr &j);
// json_free splices `o` out of its parent (if any), or clears the
// parser's root if `o` is the parser's current top-level value, and
// destroys the subtree. After this call, `o` is a dangling pointer
// that must not be used. Safe to call with nullptr.
void json_free(json_object *o);
// Splices `o` out of whatever owns it and hands it over, same detach as
// json_read_tree. See Ownership model.
json_object_ptr json_disconnect(json_object *o);
// Borrowed value for `s`. nullptr if `o` is not a hash, `s` is absent, or the
// hash is mid-parse with that value slot unfilled -- a JSON `null` is none of
// those, and comes back as a JSON_NULL node.
json_object *json_hash_get(const json_object_ptr &o, const char *s);
json_object *json_hash_get(json_object *o, const char *s);
std::string json_stringify(const json_object *o);
char *json_stringify(json_object *o);
#ifdef __cplusplus
}
#endif
#endif
+66 -26
View File
@@ -147,16 +147,17 @@ void out(std::string const &s, int type, json_object *properties) {
bool found = false;
json_object *o = json_hash_get(properties, extract);
if (o != nullptr) {
if (o != NULL) {
found = true;
if (o->type == JSON_STRING) {
extracted = sort_quote(o->string().c_str());
if (o->type == JSON_STRING || o->type == JSON_NUMBER) {
extracted = sort_quote(o->value.string.string);
} else {
// Numbers, booleans, null, and any other non-string
// values are rendered via json_stringify(); calling
// o->string() here would assert because the type-tagged
// accessor requires JSON_STRING.
extracted = sort_quote(json_stringify(o).c_str());
// Don't really know what to do about sort quoting
// for arbitrary objects
const char *out = json_stringify(o);
extracted = sort_quote(out);
free((void *) out);
}
}
@@ -232,13 +233,13 @@ void join_csv(json_object *j) {
}
json_object *properties = json_hash_get(j, "properties");
json_object *key = nullptr;
json_object *key = NULL;
if (properties != nullptr) {
if (properties != NULL) {
key = json_hash_get(properties, header[0].c_str());
}
if (key == nullptr) {
if (key == NULL) {
static bool warned = false;
if (!warned) {
fprintf(stderr, "Warning: couldn't find CSV key \"%s\" in JSON\n", header[0].c_str());
@@ -249,11 +250,13 @@ void join_csv(json_object *j) {
std::string joinkey;
if (key->type == JSON_STRING) {
joinkey = key->string();
joinkey = key->value.string.string;
} else if (key->type == JSON_NUMBER) {
joinkey = milo::dtoa_milo(key->number());
joinkey = milo::dtoa_milo(key->value.number.number);
} else {
joinkey = json_stringify(key);
const char *s = json_stringify(key);
joinkey = s;
free((void *) s);
}
if (joinkey < prev_joinkey) {
@@ -303,7 +306,14 @@ void join_csv(json_object *j) {
}
if (fields.size() > 0 && joinkey == fields[0]) {
properties->entries().reserve(properties->entries().size() + fields.size());
// This knows more about the structure of JSON objects than it ought to
// The 8 is to round up at least as much as SIZE_FOR in json_pull.c
properties->value.object.keys = (json_object **) realloc((void *) properties->value.object.keys, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *));
properties->value.object.values = (json_object **) realloc((void *) properties->value.object.values, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *));
if (properties->value.object.keys == NULL || properties->value.object.values == NULL) {
perror("realloc");
exit(EXIT_MEMORY);
}
for (size_t i = 1; i < fields.size(); i++) {
std::string k = header[i];
@@ -321,21 +331,46 @@ void join_csv(json_object *j) {
}
if (attr_type != JSON_NULL) {
json_object_ptr ko(new json_string(properties, properties->parser));
ko->string() = k;
// This knows more about the structure of JSON objects than it ought to
json_object *ko = (json_object *) malloc(sizeof(json_object));
json_object *vo = (json_object *) malloc(sizeof(json_object));
if (ko == NULL || vo == NULL) {
perror("malloc");
exit(EXIT_MEMORY);
}
ko->type = JSON_STRING;
ko->parent = properties;
ko->parser = properties->parser;
ko->value.string.string = strdup(k.c_str());
if (ko->value.string.string == NULL) {
perror("strdup");
exit(EXIT_MEMORY);
}
vo->type = attr_type;
vo->parent = properties;
vo->parser = properties->parser;
json_object_ptr vo;
if (attr_type == JSON_STRING) {
vo = json_object_ptr(new json_string(properties, properties->parser));
vo->string() = v;
vo->value.string.string = strdup(v.c_str());
if (vo->value.string.string == NULL) {
perror("strdup");
exit(EXIT_MEMORY);
}
} else if (attr_type == JSON_NUMBER) {
vo = json_object_ptr(new json_number(properties, properties->parser));
vo->set_number(atof(v.c_str()));
vo->value.number.number = atof(v.c_str());
vo->value.number.large_unsigned = 0;
vo->value.number.large_signed = 0;
} else {
abort();
}
properties->entries().push_back({std::move(ko), std::move(vo)});
properties->value.object.keys[properties->value.object.length] = ko;
properties->value.object.values[properties->value.object.length] = vo;
properties->value.object.length++;
}
}
}
@@ -348,9 +383,13 @@ struct json_join_action : json_feature_action {
join_csv(feature);
}
out(json_stringify(feature), 1, json_hash_get(feature, "properties"));
char *s = json_stringify(feature);
out(s, 1, json_hash_get(feature, "properties"));
free(s);
} else {
out(json_stringify(geometry), 2, nullptr);
char *s = json_stringify(geometry);
out(s, 2, NULL);
free(s);
}
return 1;
@@ -361,11 +400,12 @@ struct json_join_action : json_feature_action {
};
void process(FILE *fp, const char *fname) {
json_pull_ptr jp = json_begin_file(fp);
json_pull *jp = json_begin_file(fp);
json_join_action jja;
jja.fname = fname;
parse_json(&jja, jp);
json_end(jp);
}
static const struct option long_options[] = {
+33 -24
View File
@@ -587,7 +587,7 @@ struct STREAM {
}
}
json_pull_ptr json_begin() {
json_pull *json_begin() {
return ::json_begin(read_stream, this);
}
};
@@ -1832,7 +1832,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
// Plain serial reading
std::atomic<long long> layer_seq(overall_offset);
json_pull_ptr jp = fp->json_begin();
json_pull *jp = fp->json_begin();
struct serialization_state sst;
sst.fname = reading.c_str();
@@ -1859,6 +1859,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
sst.attribute_types = attribute_types;
parse_json(&sst, jp, layer, sources[layer].layer);
json_end(jp);
overall_offset = layer_seq;
checkdisk(&readers);
}
@@ -2883,10 +2884,10 @@ void set_attribute_type(std::unordered_map<std::string, int> &attribute_types, c
void set_attribute_value(const char *arg) {
if (*arg == '{') {
json_pull_ptr jp = json_begin_string(arg);
json_object_ptr o = json_read_tree(jp);
json_pull *jp = json_begin_string(arg);
json_object *o = json_read_tree(jp);
if (o == nullptr) {
if (o == NULL) {
fprintf(stderr, "%s: --set-attribute %s: %s\n", *av, arg, jp->error);
exit(EXIT_JSON);
}
@@ -2896,18 +2897,21 @@ void set_attribute_value(const char *arg) {
exit(EXIT_JSON);
}
for (size_t i = 0; i < o->entries().size(); i++) {
const auto &e = o->entries()[i];
for (size_t i = 0; i < o->value.object.length; i++) {
json_object *k = o->value.object.keys[i];
json_object *v = o->value.object.values[i];
if (e.key->type != JSON_STRING) {
if (k->type != JSON_STRING) {
fprintf(stderr, "%s: --set-attribute %s: key %zu not a string\n", *av, arg, i);
exit(EXIT_JSON);
}
serial_val val = stringify_value(e.value.get(), "json", 1, o.get());
set_attributes.emplace(e.key->string(), val);
serial_val val = stringify_value(v, "json", 1, o);
set_attributes.emplace(k->value.string.string, val);
}
json_free(o);
json_end(jp);
return;
}
@@ -2932,10 +2936,10 @@ void set_attribute_value(const char *arg) {
}
void parse_json_source(const char *arg, struct source &src) {
json_pull_ptr jp = json_begin_string(arg);
json_object_ptr o = json_read_tree(jp);
json_pull *jp = json_begin_string(arg);
json_object *o = json_read_tree(jp);
if (o == nullptr) {
if (o == NULL) {
fprintf(stderr, "%s: -L%s: %s\n", *av, arg, jp->error);
exit(EXIT_JSON);
}
@@ -2946,27 +2950,30 @@ void parse_json_source(const char *arg, struct source &src) {
}
json_object *fname = json_hash_get(o, "file");
if (fname == nullptr || fname->type != JSON_STRING) {
if (fname == NULL || fname->type != JSON_STRING) {
fprintf(stderr, "%s: -L%s: requires \"file\": filename\n", *av, arg);
exit(EXIT_JSON);
}
src.file = fname->string();
src.file = std::string(fname->value.string.string);
json_object *layer = json_hash_get(o, "layer");
if (layer != nullptr && layer->type == JSON_STRING) {
src.layer = layer->string();
if (layer != NULL && layer->type == JSON_STRING) {
src.layer = std::string(layer->value.string.string);
}
json_object *description = json_hash_get(o, "description");
if (description != nullptr && description->type == JSON_STRING) {
src.description = description->string();
if (description != NULL && description->type == JSON_STRING) {
src.description = std::string(description->value.string.string);
}
json_object *format = json_hash_get(o, "format");
if (format != nullptr && format->type == JSON_STRING) {
src.format = format->string();
if (format != NULL && format->type == JSON_STRING) {
src.format = std::string(format->value.string.string);
}
json_free(o);
json_end(jp);
}
static const struct option long_options_orig[] = {
@@ -3207,7 +3214,7 @@ int main(int argc, char **argv) {
int exclude_all = 0;
int read_parallel = 0;
int files_open_at_start;
json_object_ptr filter;
json_object *filter = NULL;
memsize = calc_memsize();
@@ -3833,7 +3840,7 @@ int main(int argc, char **argv) {
auto input_ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles
: out_dir,
maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter.get(), droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom);
maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom);
ret = std::get<0>(input_ret);
@@ -3856,7 +3863,9 @@ int main(int argc, char **argv) {
exit(EXIT_IMPOSSIBLE);
}
filter.reset();
if (filter != NULL) {
json_free(filter);
}
return ret;
}
+1 -1
View File
@@ -407,7 +407,7 @@ std::string mvt_tile::encode() {
std::string feature_string;
protozero::pbf_writer feature_writer(feature_string);
if (layers[i].features[f].type >= 0)
if (layers[i].features[f].type >= 0)
feature_writer.add_enum(3, layers[i].features[f].type);
std::vector<unsigned> sorted_tags = layers[i].features[f].tags;
+2 -2
View File
@@ -263,7 +263,7 @@ int main(int argc, char **argv) {
std::string out;
{
json_object_ptr json_filter;
json_object *json_filter = NULL;
if (filter.size() > 0) {
json_filter = parse_filter(filter.c_str());
}
@@ -289,7 +289,7 @@ int main(int argc, char **argv) {
its.push_back(std::move(t));
}
out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter.get(), preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, std::vector<mvt_layer>(), "", "", SIZE_MAX, std::vector<clipbbox>(), deduplicate_by_id);
out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, std::vector<mvt_layer>(), "", "", SIZE_MAX, std::vector<clipbbox>(), deduplicate_by_id);
}
FILE *f = fopen(outfile, "wb");
+41 -43
View File
@@ -27,7 +27,9 @@
#include "errors.hpp"
#include "thread.hpp"
extern "C" {
#include "jsonpull/jsonpull.h"
}
#include "plugin.hpp"
#include "write_json.hpp"
@@ -143,55 +145,53 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
}
// Reads from the prefilter
serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool postfilter, key_pool &key_pool) {
serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool postfilter, key_pool &key_pool) {
serial_feature sf;
while (1) {
// json_read also returns the intermediate containers of a feature
// still being assembled, so the `continue` paths leave `j` alone;
// freeing one would splice it out of its parent. `j` is only freed
// once it is a complete Feature.
json_object *j = json_read(jp);
if (j == nullptr) {
if (jp->error != nullptr) {
if (j == NULL) {
if (jp->error != NULL) {
fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error);
if (jp->root != nullptr) {
json_context(jp->root.get());
if (jp->root != NULL) {
json_context(jp->root);
} else {
fprintf(stderr, "\n");
}
exit(EXIT_JSON);
}
jp->root.reset();
json_free(jp->root);
sf.t = -1;
return sf;
}
json_object *type = json_hash_get(j, "type");
if (type == nullptr || type->type != JSON_STRING) {
if (type == NULL || type->type != JSON_STRING) {
continue;
}
if (type->string() != "Feature") {
if (strcmp(type->value.string.string, "Feature") != 0) {
continue;
}
json_object *geometry = json_hash_get(j, "geometry");
if (geometry == nullptr) {
if (geometry == NULL) {
fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line);
json_context(j);
json_free(j);
exit(EXIT_JSON);
}
json_object *properties = json_hash_get(j, "properties");
if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line);
json_context(j);
json_free(j);
exit(EXIT_JSON);
}
json_object *geometry_type = json_hash_get(geometry, "type");
if (geometry_type == nullptr) {
if (geometry_type == NULL) {
fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line);
json_context(j);
exit(EXIT_JSON);
@@ -204,7 +204,7 @@ serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, s
}
json_object *coordinates = json_hash_get(geometry, "coordinates");
if (coordinates == nullptr || coordinates->type != JSON_ARRAY) {
if (coordinates == NULL || coordinates->type != JSON_ARRAY) {
fprintf(stderr, "Filter output:%d: feature without coordinates array: ", jp->line);
json_context(j);
exit(EXIT_JSON);
@@ -212,12 +212,12 @@ serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, s
int t;
for (t = 0; t < GEOM_TYPES; t++) {
if (geometry_type->string() == geometry_names[t]) {
if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) {
break;
}
}
if (t >= GEOM_TYPES) {
fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->string().c_str());
fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string);
json_context(j);
exit(EXIT_JSON);
}
@@ -253,29 +253,29 @@ serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, s
std::string layername = "unknown";
json_object *tippecanoe = json_hash_get(j, "tippecanoe");
if (tippecanoe != nullptr) {
if (tippecanoe != NULL) {
json_object *layer = json_hash_get(tippecanoe, "layer");
if (layer != nullptr && layer->type == JSON_STRING) {
layername = layer->string();
if (layer != NULL && layer->type == JSON_STRING) {
layername = std::string(layer->value.string.string);
}
json_object *index = json_hash_get(tippecanoe, "index");
if (index != nullptr && index->type == JSON_NUMBER) {
sf.index = index->number();
if (index != NULL && index->type == JSON_NUMBER) {
sf.index = index->value.number.number;
}
json_object *sequence = json_hash_get(tippecanoe, "sequence");
if (sequence != nullptr && sequence->type == JSON_NUMBER) {
sf.seq = sequence->number();
if (sequence != NULL && sequence->type == JSON_NUMBER) {
sf.seq = sequence->value.number.number;
}
json_object *extent = json_hash_get(tippecanoe, "extent");
if (extent != nullptr && extent->type == JSON_NUMBER) {
sf.extent = extent->number();
if (extent != NULL && extent->type == JSON_NUMBER) {
sf.extent = extent->value.number.number;
}
json_object *dropped = json_hash_get(tippecanoe, "dropped");
if (dropped != nullptr && dropped->type == JSON_TRUE) {
if (dropped != NULL && dropped->type == JSON_TRUE) {
sf.dropped = FEATURE_DROPPED; // dropped
} else {
sf.dropped = FEATURE_KEPT; // kept
@@ -300,10 +300,10 @@ serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, s
}
json_object *id = json_hash_get(j, "id");
if (id != nullptr && id->type == JSON_NUMBER) {
sf.id = id->number();
if (id->large_unsigned() > 0) {
sf.id = id->large_unsigned();
if (id != NULL && id->type == JSON_NUMBER) {
sf.id = id->value.number.number;
if (id->value.number.large_unsigned > 0) {
sf.id = id->value.number.large_unsigned;
}
sf.has_id = true;
}
@@ -347,20 +347,18 @@ serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, s
}
}
if (properties->type == JSON_HASH) {
for (const auto &e : properties->entries()) {
serial_val v = stringify_value(e.value.get(), "Filter output", jp->line, j);
for (size_t i = 0; i < properties->value.object.length; i++) {
serial_val v = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j);
// Nulls can be excluded here because the expression evaluation filter
// would have already run before prefiltering
// Nulls can be excluded here because the expression evaluation filter
// would have already run before prefiltering
if (v.type != mvt_null) {
sf.full_keys.push_back(key_pool.pool(e.key->string()));
sf.full_values.push_back(v);
if (v.type != mvt_null) {
sf.full_keys.push_back(key_pool.pool(std::string(properties->value.object.keys[i]->value.string.string)));
sf.full_values.push_back(v);
if (!postfilter) {
add_to_tilestats(ts->second.tilestats, e.key->string(), v);
}
if (!postfilter) {
add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), v);
}
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
struct key_pool;
std::vector<mvt_layer> filter_layers(const char *filter, std::vector<mvt_layer> &layer, unsigned z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, int extent);
void setup_filter(const char *filter, int *write_to, int *read_from, pid_t *pid, unsigned z, unsigned x, unsigned y);
serial_feature parse_feature(json_pull_ptr &jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool filters, key_pool &key_pool);
serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, bool filters, key_pool &key_pool);
+20 -25
View File
@@ -397,9 +397,9 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) {
exit(EXIT_OPEN);
}
json_pull_ptr jp = json_begin_string(decompressed_json.c_str());
json_object_ptr o = json_read_tree(jp);
if (o == nullptr) {
json_pull *jp = json_begin_string(decompressed_json.c_str());
json_object *o = json_read_tree(jp);
if (o == NULL) {
fprintf(stderr, "%s: metadata parsing error: %s\n", fname, jp->error);
exit(EXIT_JSON);
}
@@ -415,50 +415,45 @@ sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map) {
state.nospace = true;
state.json_write_hash();
for (const auto &e : o->entries()) {
// Check before reading, since string() asserts on the type.
if (e.key->type != JSON_STRING) {
fprintf(stderr, "%s: non-string key in metadata\n", fname);
continue;
}
const std::string &key = e.key->string();
if (key == "vector_layers" && e.value->type == JSON_ARRAY) {
for (size_t i = 0; i < o->value.object.length; i++) {
const char *key = o->value.object.keys[i]->value.string.string;
if (strcmp(key, "vector_layers") == 0 && o->value.object.values[i]->type == JSON_ARRAY) {
has_json = true;
state.nospace = true;
state.json_write_string("vector_layers");
state.nospace = true;
state.json_write_json(json_stringify(e.value.get()));
} else if (key == "tilestats" && e.value->type == JSON_HASH) {
state.json_write_json(json_stringify(o->value.object.values[i]));
} else if (strcmp(key, "tilestats") == 0 && o->value.object.values[i]->type == JSON_HASH) {
has_json = true;
state.nospace = true;
state.json_write_string("tilestats");
state.nospace = true;
state.json_write_json(json_stringify(e.value.get()));
} else if (key == "strategies" && e.value->type == JSON_ARRAY) {
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(e.value.get()).c_str());
state.json_write_json(json_stringify(o->value.object.values[i]));
} else if (strcmp(key, "strategies") == 0 && o->value.object.values[i]->type == JSON_ARRAY) {
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('strategies', %Q);", json_stringify(o->value.object.values[i]));
if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err);
fprintf(stderr, "set %s in metadata: %s\n", key, err);
}
sqlite3_free(sql);
} else if (key == "tippecanoe_decisions" && e.value->type == JSON_HASH) {
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(e.value.get()).c_str());
} else if (strcmp(key, "tippecanoe_decisions") == 0 && o->value.object.values[i]->type == JSON_HASH) {
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);", json_stringify(o->value.object.values[i]));
if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err);
fprintf(stderr, "set %s in metadata: %s\n", key, err);
}
sqlite3_free(sql);
} else if (e.value->type != JSON_STRING) {
fprintf(stderr, "%s\n", key.c_str());
} else if (o->value.object.keys[i]->type != JSON_STRING || o->value.object.values[i]->type != JSON_STRING) {
fprintf(stderr, "%s\n", key);
fprintf(stderr, "%s: non-string in metadata\n", fname);
} else {
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key.c_str(), e.value->string().c_str());
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES (%Q, %Q);", key, o->value.object.values[i]->value.string.string);
if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "set %s in metadata: %s\n", key.c_str(), err);
fprintf(stderr, "set %s in metadata: %s\n", key, err);
}
sqlite3_free(sql);
}
}
json_end(jp);
state.nospace = true;
state.json_end_hash();
+56 -53
View File
@@ -43,18 +43,18 @@ int mb_geometry[GEOM_TYPES] = {
};
void json_context(json_object *j) {
std::string s = json_stringify(j);
char *s = json_stringify(j);
if (s.size() >= 500) {
s = truncate_string(s, 497);
s.append("...");
if (strlen(s) >= 500) {
snprintf(s + 497, strlen(s) + 1 - 497, "...");
}
fprintf(stderr, "in JSON object %s\n", s.c_str());
fprintf(stderr, "in JSON object %s\n", s);
free(s); // stringify
}
void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature) {
if (j == nullptr || j->type != JSON_ARRAY) {
if (j == NULL || j->type != JSON_ARRAY) {
fprintf(stderr, "%s:%d: expected array for geometry type %d: ", fname, line, t);
json_context(feature);
return;
@@ -63,7 +63,7 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *
int within = geometry_within[t];
if (within >= 0) {
size_t i;
for (i = 0; i < j->array().size(); i++) {
for (i = 0; i < j->value.array.length; i++) {
if (within == GEOM_POINT) {
if (i == 0 || mb_geometry[t] == VT_POINT) {
op = VT_MOVETO;
@@ -72,16 +72,16 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *
}
}
parse_coordinates(within, j->array()[i].get(), out, op, fname, line, feature);
parse_coordinates(within, j->value.array.array[i], out, op, fname, line, feature);
}
} else {
if (j->array().size() >= 2 && j->array()[0]->type == JSON_NUMBER && j->array()[1]->type == JSON_NUMBER) {
if (j->value.array.length >= 2 && j->value.array.array[0]->type == JSON_NUMBER && j->value.array.array[1]->type == JSON_NUMBER) {
long long x, y;
double lon = j->array()[0]->number();
double lat = j->array()[1]->number();
double lon = j->value.array.array[0]->value.number.number;
double lat = j->value.array.array[1]->value.number.number;
projection->project(lon, lat, 32, &x, &y);
if (j->array().size() > 2) {
if (j->value.array.length > 2) {
static int warned = 0;
if (!warned) {
@@ -124,12 +124,12 @@ void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *
serial_val stringify_value(json_object *value, const char *reading, int line, json_object *feature) {
serial_val sv;
if (value != nullptr) {
if (value != NULL) {
int vt = value->type;
if (vt == JSON_STRING) {
sv.type = mvt_string;
sv.s = value->string();
sv.s = value->value.string.string;
std::string err = check_utf8(sv.s);
if (err.size() > 0) {
@@ -140,12 +140,12 @@ serial_val stringify_value(json_object *value, const char *reading, int line, js
} else if (vt == JSON_NUMBER) {
sv.type = mvt_double;
if (value->large_unsigned() != 0) {
sv.s = std::to_string(value->large_unsigned());
} else if (value->large_signed() != 0) {
sv.s = std::to_string(value->large_signed());
if (value->value.number.large_unsigned != 0) {
sv.s = std::to_string(value->value.number.large_unsigned);
} else if (value->value.number.large_signed != 0) {
sv.s = std::to_string(value->value.number.large_signed);
} else {
sv.s = milo::dtoa_milo(value->number());
sv.s = milo::dtoa_milo(value->value.number.number);
}
} else if (vt == JSON_TRUE) {
sv.type = mvt_bool;
@@ -158,7 +158,9 @@ serial_val stringify_value(json_object *value, const char *reading, int line, js
sv.s = "null";
} else {
sv.type = mvt_string;
sv.s = json_stringify(value);
const char *v = json_stringify(value);
sv.s = std::string(v);
free((void *) v); // stringify
}
}
@@ -176,10 +178,10 @@ static std::vector<mvt_geometry> to_feature(drawvec &geom) {
return out;
}
std::pair<int, drawvec> parse_geometry(json_object *geometry, json_pull_ptr &jp, json_object *j,
std::pair<int, drawvec> parse_geometry(json_object *geometry, json_pull *jp, json_object *j,
int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style) {
json_object *geometry_type = json_hash_get(geometry, "type");
if (geometry_type == nullptr) {
if (geometry_type == NULL) {
fprintf(stderr, "Filter output:%d: null geometry (additional not reported): ", jp->line);
json_context(j);
exit(EXIT_JSON);
@@ -192,7 +194,7 @@ std::pair<int, drawvec> parse_geometry(json_object *geometry, json_pull_ptr &jp,
}
json_object *coordinates = json_hash_get(geometry, "coordinates");
if (coordinates == nullptr || coordinates->type != JSON_ARRAY) {
if (coordinates == NULL || coordinates->type != JSON_ARRAY) {
fprintf(stderr, "Filter output:%d: geometry without coordinates array: ", jp->line);
json_context(j);
exit(EXIT_JSON);
@@ -200,12 +202,12 @@ std::pair<int, drawvec> parse_geometry(json_object *geometry, json_pull_ptr &jp,
int t;
for (t = 0; t < GEOM_TYPES; t++) {
if (geometry_type->string() == geometry_names[t]) {
if (strcmp(geometry_type->value.string.string, geometry_names[t]) == 0) {
break;
}
}
if (t >= GEOM_TYPES) {
fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->string().c_str());
fprintf(stderr, "Filter output:%d: Can't handle geometry type %s: ", jp->line, geometry_type->value.string.string);
json_context(j);
exit(EXIT_JSON);
}
@@ -303,47 +305,47 @@ std::vector<mvt_layer> parse_layers(FILE *fp, int z, unsigned x, unsigned y, int
std::map<std::string, mvt_layer> ret;
std::shared_ptr<std::string> tile_stringpool = std::make_shared<std::string>();
json_pull_ptr jp = json_begin_file(fp);
json_pull *jp = json_begin_file(fp);
while (1) {
json_object *j = json_read(jp);
if (j == nullptr) {
if (jp->error != nullptr) {
if (j == NULL) {
if (jp->error != NULL) {
fprintf(stderr, "Filter output:%d: %s: ", jp->line, jp->error);
if (jp->root != nullptr) {
json_context(jp->root.get());
if (jp->root != NULL) {
json_context(jp->root);
} else {
fprintf(stderr, "\n");
}
exit(EXIT_JSON);
}
jp->root.reset();
json_free(jp->root);
break;
}
// Only complete Features are freed; see plugin.cpp::parse_feature.
json_object *type = json_hash_get(j, "type");
if (type == nullptr || type->type != JSON_STRING) {
if (type == NULL || type->type != JSON_STRING) {
continue;
}
if (type->string() != "Feature") {
if (strcmp(type->value.string.string, "Feature") != 0) {
continue;
}
json_object *properties = json_hash_get(j, "properties");
if (properties == nullptr || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
fprintf(stderr, "Filter output:%d: feature without properties hash: ", jp->line);
json_context(j);
json_free(j);
exit(EXIT_JSON);
}
std::string layername = "unknown";
json_object *tippecanoe = json_hash_get(j, "tippecanoe");
json_object *layer = nullptr;
if (tippecanoe != nullptr) {
json_object *layer = NULL;
if (tippecanoe != NULL) {
layer = json_hash_get(tippecanoe, "layer");
if (layer != nullptr && layer->type == JSON_STRING) {
layername = layer->string();
if (layer != NULL && layer->type == JSON_STRING) {
layername = std::string(layer->value.string.string);
}
}
@@ -358,9 +360,10 @@ std::vector<mvt_layer> parse_layers(FILE *fp, int z, unsigned x, unsigned y, int
auto l = ret.find(layername);
json_object *geometry = json_hash_get(j, "geometry");
if (geometry == nullptr) {
if (geometry == NULL) {
fprintf(stderr, "Filter output:%d: filtered feature with no geometry: ", jp->line);
json_context(j);
json_free(j);
exit(EXIT_JSON);
}
@@ -375,25 +378,23 @@ std::vector<mvt_layer> parse_layers(FILE *fp, int z, unsigned x, unsigned y, int
feature.geometry = to_feature(dv);
json_object *id = json_hash_get(j, "id");
if (id != nullptr && id->type == JSON_NUMBER) {
feature.id = id->number();
if (id->large_unsigned() > 0) {
feature.id = id->large_unsigned();
if (id != NULL && id->type == JSON_NUMBER) {
feature.id = id->value.number.number;
if (id->value.number.large_unsigned > 0) {
feature.id = id->value.number.large_unsigned;
}
feature.has_id = true;
}
if (properties->type == JSON_HASH) {
for (const auto &e : properties->entries()) {
serial_val sv = stringify_value(e.value.get(), "Filter output", jp->line, j);
for (size_t i = 0; i < properties->value.object.length; i++) {
serial_val sv = stringify_value(properties->value.object.values[i], "Filter output", jp->line, j);
// Nulls can be excluded here because this is the postfilter
// and it is nearly time to create the vector representation
// Nulls can be excluded here because this is the postfilter
// and it is nearly time to create the vector representation
if (sv.type != mvt_null) {
mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool);
l->second.tag(feature, e.key->string(), v);
}
if (sv.type != mvt_null) {
mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str(), tile_stringpool);
l->second.tag(feature, std::string(properties->value.object.keys[i]->value.string.string), v);
}
}
@@ -403,6 +404,8 @@ std::vector<mvt_layer> parse_layers(FILE *fp, int z, unsigned x, unsigned y, int
json_free(j);
}
json_end(jp);
std::vector<mvt_layer> final;
for (auto a : ret) {
final.push_back(a.second);
+1 -1
View File
@@ -12,7 +12,7 @@ extern int mb_geometry[GEOM_TYPES];
void json_context(json_object *j);
void parse_coordinates(int t, json_object *j, drawvec &out, int op, const char *fname, int line, json_object *feature);
std::pair<int, drawvec> parse_geometry(json_object *geometry, json_pull_ptr &jp, json_object *j,
std::pair<int, drawvec> parse_geometry(json_object *geometry, json_pull *jp, json_object *j,
int z, int x, int y, long long extent, bool fix_longitudes, bool mvt_style);
std::vector<mvt_layer> parse_layers(FILE *fp, int z, unsigned x, unsigned y, int extent, bool fix_longitudes);
+61 -54
View File
@@ -336,7 +336,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
outfeature.geometry[i].x = outfeature.geometry[i].x * outlayer.extent / layer.extent;
outfeature.geometry[i].y = outfeature.geometry[i].y * outlayer.extent / layer.extent;
}
}
}
for (auto const &g : outfeature.geometry) {
if (g.op == mvt_moveto || g.op == mvt_lineto) {
@@ -966,44 +966,44 @@ void dispatch_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<
}
void handle_strategies(const unsigned char *s, std::vector<strategy> *st) {
json_pull_ptr jp = json_begin_string((const char *) s);
json_object_ptr o = json_read_tree(jp);
json_pull *jp = json_begin_string((const char *) s);
json_object *o = json_read_tree(jp);
if (o != nullptr && o->type == JSON_ARRAY) {
for (size_t i = 0; i < o->array().size(); i++) {
const json_object_ptr &h = o->array()[i];
if (o != NULL && o->type == JSON_ARRAY) {
for (size_t i = 0; i < o->value.array.length; i++) {
json_object *h = o->value.array.array[i];
if (h->type == JSON_HASH) {
for (size_t j = 0; j < h->entries().size(); j++) {
const auto &kv = h->entries()[j];
for (size_t j = 0; j < h->value.object.length; j++) {
json_object *k = h->value.object.keys[j];
json_object *v = h->value.object.values[j];
if (kv.key->type != JSON_STRING) {
if (k->type != JSON_STRING) {
fprintf(stderr, "Key %zu of %zu is not a string: %s\n", j, i, s);
} else if (kv.value->type != JSON_NUMBER) {
} else if (v->type != JSON_NUMBER) {
fprintf(stderr, "Value %zu of %zu is not a number: %s\n", j, i, s);
} else {
if (i >= st->size()) {
st->resize(i + 1);
}
const std::string &key = kv.key->string();
if (key == "dropped_by_rate") {
(*st)[i].dropped_by_rate += kv.value->number();
} else if (key == "dropped_by_gamma") {
(*st)[i].dropped_by_gamma += kv.value->number();
} else if (key == "dropped_as_needed") {
(*st)[i].dropped_as_needed += kv.value->number();
} else if (key == "coalesced_as_needed") {
(*st)[i].coalesced_as_needed += kv.value->number();
} else if (key == "truncated_zooms") {
(*st)[i].truncated_zooms += kv.value->number();
} else if (key == "detail_reduced") {
(*st)[i].detail_reduced += kv.value->number();
} else if (key == "tiny_polygons") {
(*st)[i].tiny_polygons += kv.value->number();
} else if (key == "tile_size_desired") {
(*st)[i].tile_size += kv.value->number();
} else if (key == "feature_count_desired") {
(*st)[i].feature_count += kv.value->number();
if (strcmp(k->value.string.string, "dropped_by_rate") == 0) {
(*st)[i].dropped_by_rate += v->value.number.number;
} else if (strcmp(k->value.string.string, "dropped_by_gamma") == 0) {
(*st)[i].dropped_by_gamma += v->value.number.number;
} else if (strcmp(k->value.string.string, "dropped_as_needed") == 0) {
(*st)[i].dropped_as_needed += v->value.number.number;
} else if (strcmp(k->value.string.string, "coalesced_as_needed") == 0) {
(*st)[i].coalesced_as_needed += v->value.number.number;
} else if (strcmp(k->value.string.string, "truncated_zooms") == 0) {
(*st)[i].truncated_zooms += v->value.number.number;
} else if (strcmp(k->value.string.string, "detail_reduced") == 0) {
(*st)[i].detail_reduced += v->value.number.number;
} else if (strcmp(k->value.string.string, "tiny_polygons") == 0) {
(*st)[i].tiny_polygons += v->value.number.number;
} else if (strcmp(k->value.string.string, "tile_size_desired") == 0) {
(*st)[i].tile_size += v->value.number.number;
} else if (strcmp(k->value.string.string, "feature_count_desired") == 0) {
(*st)[i].feature_count += v->value.number.number;
}
}
}
@@ -1011,19 +1011,22 @@ void handle_strategies(const unsigned char *s, std::vector<strategy> *st) {
fprintf(stderr, "Element %zu is not a hash: %s\n", i, s);
}
}
json_free(o);
}
json_end(jp);
}
void handle_vector_layers(json_object *vector_layers, std::map<std::string, layermap_entry> &layermap, std::map<std::string, std::string> &attribute_descriptions) {
if (vector_layers != nullptr && vector_layers->type == JSON_ARRAY) {
for (size_t i = 0; i < vector_layers->array().size(); i++) {
if (vector_layers->array()[i]->type == JSON_HASH) {
json_object *id = json_hash_get(vector_layers->array()[i].get(), "id");
json_object *desc = json_hash_get(vector_layers->array()[i].get(), "description");
if (vector_layers != NULL && vector_layers->type == JSON_ARRAY) {
for (size_t i = 0; i < vector_layers->value.array.length; i++) {
if (vector_layers->value.array.array[i]->type == JSON_HASH) {
json_object *id = json_hash_get(vector_layers->value.array.array[i], "id");
json_object *desc = json_hash_get(vector_layers->value.array.array[i], "description");
if (id != nullptr && desc != nullptr && id->type == JSON_STRING && desc->type == JSON_STRING) {
const std::string &sid = id->string();
const std::string &sdesc = desc->string();
if (id != NULL && desc != NULL && id->type == JSON_STRING && desc->type == JSON_STRING) {
std::string sid = id->value.string.string;
std::string sdesc = desc->value.string.string;
if (sdesc.size() != 0) {
auto f = layermap.find(sid);
@@ -1033,18 +1036,17 @@ void handle_vector_layers(json_object *vector_layers, std::map<std::string, laye
}
}
json_object *fields = json_hash_get(vector_layers->array()[i].get(), "fields");
if (fields != nullptr && fields->type == JSON_HASH) {
for (const auto &e : fields->entries()) {
if (e.key != nullptr && e.key->type == JSON_STRING &&
e.value != nullptr && e.value->type == JSON_STRING) {
const std::string &desc2 = e.value->string();
json_object *fields = json_hash_get(vector_layers->value.array.array[i], "fields");
if (fields != NULL && fields->type == JSON_HASH) {
for (size_t j = 0; j < fields->value.object.length; j++) {
if (fields->value.object.keys[j]->type == JSON_STRING && fields->value.object.values[j]->type) {
const char *desc2 = fields->value.object.values[j]->value.string.string;
if (desc2 != "Number" &&
desc2 != "String" &&
desc2 != "Boolean" &&
desc2 != "Mixed") {
attribute_descriptions.insert(std::pair<std::string, std::string>(e.key->string(), desc2));
if (strcmp(desc2, "Number") != 0 &&
strcmp(desc2, "String") != 0 &&
strcmp(desc2, "Boolean") != 0 &&
strcmp(desc2, "Mixed") != 0) {
attribute_descriptions.insert(std::pair<std::string, std::string>(fields->value.object.keys[j]->value.string.string, desc2));
}
}
}
@@ -1204,14 +1206,17 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
const unsigned char *s = sqlite3_column_text(stmt, 0);
if (s != NULL) {
json_pull_ptr jp = json_begin_string((const char *) s);
json_object_ptr o = json_read_tree(jp);
json_pull *jp = json_begin_string((const char *) s);
json_object *o = json_read_tree(jp);
if (o != nullptr && o->type == JSON_HASH) {
if (o != NULL && o->type == JSON_HASH) {
json_object *vector_layers = json_hash_get(o, "vector_layers");
handle_vector_layers(vector_layers, layermap, attribute_descriptions);
json_free(o);
}
json_end(jp);
}
}
@@ -1335,7 +1340,7 @@ int main(int argc, char **argv) {
int force = 0;
int ifmatched = 0;
int filearg = 0;
json_object_ptr filter;
json_object *filter = NULL;
std::string join_sqlite_fname;
@@ -1616,7 +1621,7 @@ int main(int argc, char **argv) {
std::string generator_options;
std::vector<strategy> strategies;
decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter.get(), attribute_descriptions, generator_options, &strategies);
decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies);
if (set_attribution.size() != 0) {
attribution = set_attribution;
@@ -1668,7 +1673,9 @@ int main(int argc, char **argv) {
mbtiles_close(outdb, argv[0]);
}
filter.reset();
if (filter != NULL) {
json_free(filter);
}
if (pmtiles_has_suffix(out_mbtiles)) {
mbtiles_map_image_to_pmtiles(out_mbtiles, m, !pC, quiet, false);
+12 -8
View File
@@ -48,7 +48,11 @@
#include "attribute.hpp"
#include "thread.hpp"
#include "shared_borders.hpp"
extern "C" {
#include "jsonpull/jsonpull.h"
}
#include "plugin.hpp"
#define CMD_BITS 3
@@ -823,7 +827,7 @@ static double choose_minattribute(std::vector<double> &attribute_values, double
if (descending) {
// For descending: drop features > threshold, keep features <= threshold
// ix points at the last value to keep
size_t ix = (size_t) ((attribute_values.size() - 1) * f);
size_t ix = (size_t)((attribute_values.size() - 1) * f);
while (ix > 0 && attribute_values[ix] >= existing_attribute) {
ix--;
}
@@ -836,7 +840,7 @@ static double choose_minattribute(std::vector<double> &attribute_values, double
} else {
// For ascending: drop features < threshold, keep features >= threshold
// ix points at the first value to keep
size_t ix = (size_t) ceil((double) (attribute_values.size() - 1) * (1 - f));
size_t ix = (size_t)ceil((double)(attribute_values.size() - 1) * (1 - f));
if (ix >= attribute_values.size()) {
ix = attribute_values.size() - 1;
}
@@ -1792,7 +1796,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
pthread_t prefilter_writer;
run_prefilter_args rpa; // here so it stays in scope until joined
FILE *prefilter_read_fp = NULL;
json_pull_ptr prefilter_jp;
json_pull *prefilter_jp = NULL;
if (z < minzoom) {
prefilter = NULL;
@@ -2128,8 +2132,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (attr_valid) {
add_sample_to(attribute_values, attr_numeric, attribute_values_increment, seq);
bool should_drop = arg->drop_by_attribute_descending
? (minattribute != HUGE_VAL && attr_numeric > minattribute)
: (minattribute != -HUGE_VAL && attr_numeric < minattribute);
? (minattribute != HUGE_VAL && attr_numeric > minattribute)
: (minattribute != -HUGE_VAL && attr_numeric < minattribute);
if (should_drop) {
can_stop_early = false;
if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, strategy, drop_rest, arg->attribute_accum, key_pool)) {
@@ -2797,7 +2801,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
} else if (additional[A_DROP_BY_ATTRIBUTE_AS_NEEDED]) {
minattribute_fraction = minattribute_fraction *
adjusted_max_tile_features / adjusted_feature_count * 0.75;
adjusted_max_tile_features / adjusted_feature_count * 0.75;
if (minattribute_fraction > 0.80) {
if (!quiet) {
fprintf(stderr,
@@ -3516,8 +3520,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
again = true;
}
bool attr_propagate = drop_by_attribute_descending
? args[thread].minattribute_out < zoom_minattribute
: args[thread].minattribute_out > zoom_minattribute;
? args[thread].minattribute_out < zoom_minattribute
: args[thread].minattribute_out > zoom_minattribute;
if (attr_propagate) {
zoom_minattribute = args[thread].minattribute_out;
again = true;
+1 -1
View File
@@ -62,7 +62,7 @@ struct strategy {
// long long write_tile(char **geom, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector<std::string> const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending);
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, int basezoom, double droprate, std::vector<std::string> const &unidecode_data, std::string const *drop_by_attribute_as_needed_attribute, bool drop_by_attribute_descending);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);
+2 -361
View File
@@ -6,7 +6,6 @@
#include "mvt.hpp"
#include "projection.hpp"
#include "geometry.hpp"
#include "jsonpull/jsonpull.h"
#include <unistd.h>
#include <limits.h>
@@ -138,364 +137,6 @@ TEST_CASE("line_is_too_small") {
REQUIRE(line_is_too_small(dv, 0, 10));
}
// A high surrogate followed by a non-surrogate used to be combined into one
// code point, because the range check tested the outer-loop byte instead of
// the parsed code unit. The stale surrogate should come out as standalone
// CESU-8, then U+E000 encoded normally.
TEST_CASE("jsonpull surrogate-pair regression", "[jsonpull][surrogate]") {
json_pull_ptr jp = json_begin_string("\"\\uD83D\\uE000\"");
json_object_ptr o = json_read_tree(jp);
REQUIRE(jp->error == nullptr);
REQUIRE(o != nullptr);
REQUIRE(o->type == JSON_STRING);
const std::string expected = "\xED\xA0\xBD\xEE\x80\x80";
REQUIRE(o->string() == expected);
// Sanity check: the buggy output (a single 4-byte UTF-8 sequence for
// U+1F400) must not be what we got.
const std::string buggy = "\xF0\x9F\x90\x80";
REQUIRE(o->string() != buggy);
}
// What geojson-loop does: free each feature once serialized, so they do not
// accumulate while the rest of the document is parsed.
//
// This shape cannot catch the array-splicing bug, at any size: json_read hands
// back each container as it completes, so the node freed here is always the
// last-added element of its parent, which the old memmove got right by moving
// zero bytes. "json_free prunes a non-final element" below covers that.
TEST_CASE("json_free prunes a subtree from its parent", "[jsonpull][memory]") {
json_pull_ptr jp = json_begin_string("[[1, 2], [3, 4], [5, 6]]");
json_object *outer = nullptr;
int arrays_seen = 0;
json_object *j;
while ((j = json_read(jp)) != nullptr) {
if (j->type != JSON_ARRAY) {
continue;
}
arrays_seen++;
if (arrays_seen == 2) {
// This is [3, 4]; verify, then ask the parser to drop it.
REQUIRE(j->array().size() == 2);
REQUIRE(j->array()[0]->number() == 3);
REQUIRE(j->array()[1]->number() == 4);
json_free(j);
} else if (j->parent == nullptr) {
// The completed outer array; the parser still owns it
// via jp->root, so the borrowed pointer stays valid.
outer = j;
break;
}
}
REQUIRE(outer != nullptr);
REQUIRE(outer->type == JSON_ARRAY);
REQUIRE(outer->array().size() == 2);
// First surviving element: [1, 2].
REQUIRE(outer->array()[0]->type == JSON_ARRAY);
REQUIRE(outer->array()[0]->array().size() == 2);
REQUIRE(outer->array()[0]->array()[0]->number() == 1);
REQUIRE(outer->array()[0]->array()[1]->number() == 2);
// Second surviving element (previously third): [5, 6].
REQUIRE(outer->array()[1]->type == JSON_ARRAY);
REQUIRE(outer->array()[1]->array().size() == 2);
REQUIRE(outer->array()[1]->array()[0]->number() == 5);
REQUIRE(outer->array()[1]->array()[1]->number() == 6);
}
// A top-level value has no parent, so json_free has to drop the parser's
// reference instead. jp->root being empty afterwards is also proof the subtree
// was destroyed, since jp->root was its only owner.
TEST_CASE("json_free releases a top-level value held by the parser", "[jsonpull][memory]") {
json_pull_ptr jp = json_begin_string(R"({"a": 1, "b": [2, 3]})");
// json_read streams atoms first (1, 2, 3, [2,3], ...); the top-level
// hash is returned by the final `}` token.
json_object *top = nullptr;
json_object *j;
while ((j = json_read(jp)) != nullptr) {
if (j->parent == nullptr) {
top = j;
break;
}
}
REQUIRE(top != nullptr);
REQUIRE(top->type == JSON_HASH);
REQUIRE(jp->root.get() == top);
json_free(top);
// top is dangling now; do not dereference.
REQUIRE(jp->root == nullptr);
}
// Nothing in tippecanoe calls json_disconnect today -- the filter loaders get
// the same guarantee from json_read_tree -- so cover it here rather than leave
// a documented ownership primitive untested.
TEST_CASE("json_disconnect hands a subtree to the caller", "[jsonpull][ownership]") {
json_object_ptr taken;
json_object *outer = nullptr;
json_pull_ptr jp = json_begin_string("[[1, 2], [3, 4]]");
int arrays_seen = 0;
json_object *j;
while ((j = json_read(jp)) != nullptr) {
if (j->type != JSON_ARRAY) {
continue;
}
arrays_seen++;
if (arrays_seen == 2) {
// This is [3, 4]; take it away from the enclosing array.
taken = json_disconnect(j);
REQUIRE(taken != nullptr);
REQUIRE(taken.get() == j);
} else if (j->parent == nullptr) {
outer = j;
break;
}
}
// The outer array is left holding only the element we didn't take,
// and the parser is still the owner of it.
REQUIRE(outer != nullptr);
REQUIRE(jp->root.get() == outer);
REQUIRE(outer->array().size() == 1);
REQUIRE(outer->array()[0]->array().size() == 2);
REQUIRE(outer->array()[0]->array()[0]->number() == 1);
REQUIRE(outer->array()[0]->array()[1]->number() == 2);
// The detached subtree holds no pointers into the parser...
REQUIRE(taken->parser == nullptr);
REQUIRE(taken->array()[0]->parser == nullptr);
REQUIRE(taken->array()[1]->parser == nullptr);
// ...and its root no longer points out at the array it was spliced
// from, but the parent links *within* it are left intact so the tree
// stays navigable upwards.
REQUIRE(taken->parent == nullptr);
REQUIRE(taken->array()[0]->parent == taken.get());
REQUIRE(taken->array()[1]->parent == taken.get());
// So the subtree stays valid once the parser, and the tree the parser
// still owns, are destroyed.
jp.reset();
// outer is dangling now; do not dereference.
REQUIRE(taken->type == JSON_ARRAY);
REQUIRE(taken->array().size() == 2);
REQUIRE(taken->array()[0]->number() == 3);
REQUIRE(taken->array()[1]->number() == 4);
}
// json_free finds a node's owner through o->parent, so clearing the interior
// parent links on detach would make this a silent no-op.
TEST_CASE("json_free prunes an interior node of a detached tree", "[jsonpull][ownership]") {
json_pull_ptr jp = json_begin_string("[[1, 2], [3, 4], [5, 6]]");
json_object_ptr tree = json_read_tree(jp);
REQUIRE(tree != nullptr);
REQUIRE(tree->type == JSON_ARRAY);
REQUIRE(tree->array().size() == 3);
// Destroy the parser first, so this is unambiguously operating on a
// tree that no longer has one.
jp.reset();
REQUIRE(tree->parser == nullptr);
json_object *drop = tree->array()[1].get();
REQUIRE(drop->array()[0]->number() == 3);
REQUIRE(drop->parent == tree.get());
json_free(drop);
// drop is dangling now; do not dereference.
// Had the parent links been cleared on detach, json_free would not
// have found an owner to splice the node out of, and the array would
// still have three elements.
REQUIRE(tree->array().size() == 2);
REQUIRE(tree->array()[0]->array()[0]->number() == 1);
REQUIRE(tree->array()[1]->array()[0]->number() == 5);
}
// Not a removal: the key stays with a JSON_NULL stand-in so the surrounding
// pairs keep their alignment, and the entry goes only when both halves do.
TEST_CASE("json_free of a hash value leaves a null placeholder", "[jsonpull][ownership]") {
json_pull_ptr jp = json_begin_string(R"({"keep": 1, "drop": [2, 3]})");
json_object_ptr tree = json_read_tree(jp);
REQUIRE(tree != nullptr);
REQUIRE(tree->entries().size() == 2);
json_object *drop = json_hash_get(tree, "drop");
REQUIRE(drop != nullptr);
REQUIRE(drop->type == JSON_ARRAY);
json_free(drop);
// drop is dangling now; do not dereference.
// The key survives, now paired with a null rather than the array.
REQUIRE(tree->entries().size() == 2);
json_object *after = json_hash_get(tree, "drop");
REQUIRE(after != nullptr);
REQUIRE(after->type == JSON_NULL);
// The neighbouring pair is untouched.
REQUIRE(json_hash_get(tree, "keep") != nullptr);
REQUIRE(json_hash_get(tree, "keep")->number() == 1);
}
// The array-splicing fix. The old code passed an element count to memmove
// where a byte count was wanted, so pruning element 0 of eight left
// arr[0] == arr[1] -- one node owned twice -- and dropped the last element.
// Asserting each survivor's identity is what discriminates; checking only the
// size would not.
TEST_CASE("json_free prunes a non-final element", "[jsonpull][ownership]") {
json_pull_ptr jp = json_begin_string("[[1], [2], [3], [4], [5], [6], [7], [8]]");
json_object_ptr tree = json_read_tree(jp);
REQUIRE(tree != nullptr);
REQUIRE(tree->type == JSON_ARRAY);
REQUIRE(tree->array().size() == 8);
json_free(tree->array()[0].get());
// Every survivor keeps its identity, in order, and nothing is aliased.
REQUIRE(tree->array().size() == 7);
for (size_t i = 0; i < tree->array().size(); i++) {
json_object *e = tree->array()[i].get();
REQUIRE(e->type == JSON_ARRAY);
REQUIRE(e->array().size() == 1);
REQUIRE(e->array()[0]->number() == (double) (i + 2));
if (i + 1 < tree->array().size()) {
REQUIRE(e != tree->array()[i + 1].get());
}
}
}
// The mirror of the value case above.
TEST_CASE("json_free of a hash key, then of both halves", "[jsonpull][ownership]") {
json_pull_ptr jp = json_begin_string(R"({"a": 1, "b": 2, "c": 3})");
json_object_ptr tree = json_read_tree(jp);
REQUIRE(tree != nullptr);
REQUIRE(tree->entries().size() == 3);
// Free the key of the middle pair. The entry stays, with a null key,
// and its value is still reachable positionally.
json_free(tree->entries()[1].key.get());
REQUIRE(tree->entries().size() == 3);
REQUIRE(tree->entries()[1].key->type == JSON_NULL);
REQUIRE(tree->entries()[1].value->number() == 2);
// The neighbours are untouched, and the now-keyless pair is no longer
// findable by name.
REQUIRE(json_hash_get(tree, "b") == nullptr);
REQUIRE(json_hash_get(tree, "a")->number() == 1);
REQUIRE(json_hash_get(tree, "c")->number() == 3);
// Freeing the other half too retires the whole entry.
json_free(tree->entries()[1].value.get());
REQUIRE(tree->entries().size() == 2);
REQUIRE(json_hash_get(tree, "a")->number() == 1);
REQUIRE(json_hash_get(tree, "c")->number() == 3);
}
// What the filter loaders and -L / -E do. Each detached tree has to survive
// the next json_read_tree call and the parser's destruction.
TEST_CASE("repeated json_read_tree on a line-delimited stream", "[jsonpull][ownership]") {
json_pull_ptr jp = json_begin_string("{\"n\": 1}\n{\"n\": 2}\n{\"n\": 3}\n");
std::vector<json_object_ptr> trees;
for (int i = 0; i < 3; i++) {
json_object_ptr t = json_read_tree(jp);
REQUIRE(t != nullptr);
REQUIRE(t->type == JSON_HASH);
// Reading the next tree must not disturb the ones already taken.
REQUIRE(json_hash_get(t, "n")->number() == i + 1);
trees.push_back(std::move(t));
}
REQUIRE(json_read_tree(jp) == nullptr);
// All three outlive the parser they came from.
jp.reset();
for (int i = 0; i < 3; i++) {
REQUIRE(trees[i]->parser == nullptr);
REQUIRE(json_hash_get(trees[i], "n")->number() == i + 1);
}
}
// What json_context() prints on the error paths: a hash whose last key has no
// value yet renders that slot as "...".
TEST_CASE("json_stringify of a partially-parsed tree", "[jsonpull][stringify]") {
json_pull_ptr jp = json_begin_string("{\"a\": [1, 2], \"b\":");
// Read until the parser runs out of input mid-hash.
while (json_read(jp) != nullptr) {
;
}
REQUIRE(jp->error != nullptr);
REQUIRE(jp->root != nullptr);
std::string s = json_stringify(jp->root.get());
REQUIRE(s == "{\"a\":[1,2],\"b\":...}");
}
// Values are std::string now, so an embedded NUL is legal and stringify has to
// walk past it rather than stop the way a c_str() loop would.
TEST_CASE("json_stringify keeps text after an embedded NUL", "[jsonpull][stringify]") {
json_pull_ptr jp = json_begin_string("\"a\\u0000b\"");
json_object_ptr o = json_read_tree(jp);
REQUIRE(o != nullptr);
REQUIRE(o->type == JSON_STRING);
REQUIRE(o->string().size() == 3);
std::string s = json_stringify(o.get());
REQUIRE(s == "\"a\\u0000b\"");
}
// U+FFFF used to fall through the `< 0xFFFF` test into the four-byte branch
// and come out as the overlong F0 8F BF BF, which check_utf8 does not catch.
TEST_CASE("jsonpull encodes U+FFFF as three bytes", "[jsonpull][utf8]") {
json_pull_ptr jp = json_begin_string("\"a\\uFFFFb\"");
json_object_ptr o = json_read_tree(jp);
REQUIRE(jp->error == nullptr);
REQUIRE(o != nullptr);
REQUIRE(o->type == JSON_STRING);
REQUIRE(o->string() ==
"a\xEF\xBF\xBF"
"b");
REQUIRE(o->string() !=
"a\xF0\x8F\xBF\xBF"
"b");
// The boundary below it, and a genuine supplementary code point built
// from a surrogate pair, both keep their existing encodings.
json_pull_ptr jp2 = json_begin_string("\"\\uFFFE\"");
json_object_ptr o2 = json_read_tree(jp2);
REQUIRE(o2 != nullptr);
REQUIRE(o2->string() == "\xEF\xBF\xBE");
json_pull_ptr jp3 = json_begin_string("\"\\uD83D\\uDC00\"");
json_object_ptr o3 = json_read_tree(jp3);
REQUIRE(o3 != nullptr);
REQUIRE(o3->string() == "\xF0\x9F\x90\x80");
}
TEST_CASE("Polygon cleaning drops a hole that no ring can parent", "[wagyu]") {
// Two mutually reversed self-intersecting rings whose union leaves a hole
// that wagyu's topology correction cannot assign to any surviving parent
@@ -504,8 +145,8 @@ TEST_CASE("Polygon cleaning drops a hole that no ring can parent", "[wagyu]") {
// through the "Could not properly place hole to a parent." handler in
// clean_or_clip_poly instead of returning.
static const std::vector<std::vector<std::pair<long long, long long>>> rings = {
{{0, 5}, {5, 4}, {5, 1}, {4, 4}, {4, 2}, {7, 1}, {0, 5}},
{{0, 5}, {7, 1}, {4, 2}, {4, 4}, {5, 1}, {5, 4}, {0, 0}, {0, 5}},
{{0, 5}, {5, 4}, {5, 1}, {4, 4}, {4, 2}, {7, 1}, {0, 5}},
{{0, 5}, {7, 1}, {4, 2}, {4, 4}, {5, 1}, {5, 4}, {0, 0}, {0, 5}},
};
drawvec geom;
+1 -1
View File
@@ -1,6 +1,6 @@
#ifndef VERSION_HPP
#define VERSION_HPP
#define VERSION "v2.82.0"
#define VERSION "v2.81.0"
#endif