mirror of
https://github.com/felt/tippecanoe.git
synced 2026-11-22 16:31:18 +01:00
Compare commits
9
Commits
main
...
flatgeobuf-fix
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
538a5feab4 | ||
|
|
a44279ee87 | ||
|
|
12a6242ef1 | ||
|
|
5cbec4e61f | ||
|
|
c94fb1f09e | ||
|
|
5f5fcb36a9 | ||
|
|
3f802920bb | ||
|
|
9f831bfca5 | ||
|
|
2419350e09 |
@@ -97,10 +97,27 @@ suffixes = json json.gz
|
||||
# Don't test overflow with geobuf, because it fails (https://github.com/mapbox/geobuf/issues/87)
|
||||
# Don't test stringids with geobuf, because it fails
|
||||
nogeobuf = tests/overflow/out/-z0.json $(wildcard tests/stringid/out/*.json)
|
||||
|
||||
# Don't test attribute-type with geobuf, because ogr2ogr doesn't support different types for the same attribute between features
|
||||
noflatgeobuf = $(wildcard tests/attribute-type/out/*.json)
|
||||
# These inputs all have feature IDs, which aren't supported by flatgeobuf
|
||||
noflatgeobuf += $(foreach dir,coalesce-id feature-filter id overflow stringid wyalkatchem,$(wildcard tests/$(dir)/out/*.json))
|
||||
# These inputs all have null attributes, which aren't supported by flatgeobuf
|
||||
noflatgeobuf += $(foreach dir,attribute-type border dateline epsg-3857 feature-filter islands ne_110m_admin_1_states_provinces_lines ne_110m_populated_places tl_2018_51685_roads wraparound,$(wildcard tests/$(dir)/out/*.json))
|
||||
# These inputs have weird GeoJSON geometry types, which aren't supported by flatgeobuf
|
||||
noflatgeobuf += $(foreach dir,geometry,$(wildcard tests/$(dir)/out/*.json))
|
||||
# Feature order difference, probably from coordinate rounding?
|
||||
noflatgeobuf += $(foreach dir,grid-unaligned knox multilayer,$(wildcard tests/$(dir)/out/*.json))
|
||||
# Per-feature layer declarations
|
||||
noflatgeobuf += $(foreach dir,allow-existing csv feature-filter join-population layer-json longlayer muni pbf raw-tiles,$(wildcard tests/$(dir)/out/*.json))
|
||||
# Per-feature minzooms or maxzooms
|
||||
noflatgeobuf += $(foreach dir,allow-existing csv dateline join-population layer-json minzoom onefeature raw-tiles,$(wildcard tests/$(dir)/out/*.json))
|
||||
|
||||
geobuf-test: tippecanoe-json-tool $(addsuffix .checkbuf,$(filter-out $(nogeobuf),$(TESTS)))
|
||||
flatgeobuf-test: tippecanoe-json-tool $(addsuffix .checkflatbuf,$(filter-out $(noflatgeobuf),$(TESTS)))
|
||||
|
||||
# For quicker address sanitizer build, hope that regular JSON parsing is tested enough by parallel and join tests
|
||||
fewer-tests: tippecanoe tippecanoe-decode geobuf-test raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit
|
||||
fewer-tests: tippecanoe tippecanoe-decode flatgeobuf-test geobuf-test raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit
|
||||
|
||||
# XXX Use proper makefile rules instead of a for loop
|
||||
%.json.checkbuf:
|
||||
@@ -111,6 +128,15 @@ fewer-tests: tippecanoe tippecanoe-decode geobuf-test raw-tiles-test parallel-te
|
||||
cmp $@.out $(patsubst %.checkbuf,%,$@)
|
||||
rm $@.out $@.mbtiles
|
||||
|
||||
# XXX Use proper makefile rules instead of a for loop
|
||||
%.json.checkflatbuf:
|
||||
for i in $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json); do rm -f $$i.fgb; ./tippecanoe-json-tool -w $$i > $$i.clean && ogr2ogr -preserve_fid $$i.fgb $$i.clean; done
|
||||
for i in $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json.gz); do rm -f $$i.fgb; gzip -dc $$i | ./tippecanoe-json-tool -w > $$i.clean && ogr2ogr -preserve_fid $$i.fgb $$i.clean; done
|
||||
./tippecanoe -q -a@ -f -o $@.mbtiles $(subst @,:,$(subst %,/,$(subst _, ,$(patsubst %.json.checkflatbuf,%,$(word 4,$(subst /, ,$@)))))) $(foreach suffix,$(suffixes),$(addsuffix .fgb,$(sort $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.$(suffix))))) < /dev/null
|
||||
./tippecanoe-decode -x generator $@.mbtiles | sed 's/checkflatbuf/check/g' | sed 's/\.fgb//g' > $@.out
|
||||
cmp $@.out $(patsubst %.checkflatbuf,%,$@)
|
||||
rm $@.out $@.mbtiles
|
||||
|
||||
parallel-test:
|
||||
mkdir -p tests/parallel
|
||||
perl -e 'for ($$i = 0; $$i < 20; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; $$k = rand(1); $$v = rand(1); print "{ \"type\": \"Feature\", \"properties\": { \"yes\": \"no\", \"who\": 1, \"$$k\": \"$$v\" }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in1.json
|
||||
|
||||
+172
-153
@@ -8,43 +8,42 @@
|
||||
#include "main.hpp"
|
||||
#include "errors.hpp"
|
||||
|
||||
static constexpr uint8_t magicbytes[8] = { 0x66, 0x67, 0x62, 0x03, 0x66, 0x67, 0x62, 0x01 };
|
||||
static constexpr uint8_t magicbytes[8] = {0x66, 0x67, 0x62, 0x03, 0x66, 0x67, 0x62, 0x01};
|
||||
|
||||
struct NodeItem {
|
||||
double minX;
|
||||
double minY;
|
||||
double maxX;
|
||||
double maxY;
|
||||
uint64_t offset;
|
||||
double minX;
|
||||
double minY;
|
||||
double maxX;
|
||||
double maxY;
|
||||
uint64_t offset;
|
||||
};
|
||||
|
||||
// copied from https://github.com/flatgeobuf/flatgeobuf/blob/master/src/cpp/packedrtree.cpp#L365
|
||||
uint64_t PackedRTreeSize(const uint64_t numItems, const uint16_t nodeSize)
|
||||
{
|
||||
if (nodeSize < 2)
|
||||
throw std::invalid_argument("Node size must be at least 2");
|
||||
if (numItems == 0)
|
||||
throw std::invalid_argument("Number of items must be greater than 0");
|
||||
const uint16_t nodeSizeMin = std::min(std::max(nodeSize, static_cast<uint16_t>(2)), static_cast<uint16_t>(65535));
|
||||
// limit so that resulting size in bytes can be represented by uint64_t
|
||||
if (numItems > static_cast<uint64_t>(1) << 56)
|
||||
throw std::overflow_error("Number of items must be less than 2^56");
|
||||
uint64_t n = numItems;
|
||||
uint64_t numNodes = n;
|
||||
do {
|
||||
n = (n + nodeSizeMin - 1) / nodeSizeMin;
|
||||
numNodes += n;
|
||||
} while (n != 1);
|
||||
return numNodes * sizeof(NodeItem);
|
||||
uint64_t PackedRTreeSize(const uint64_t numItems, const uint16_t nodeSize) {
|
||||
if (nodeSize < 2)
|
||||
throw std::invalid_argument("Node size must be at least 2");
|
||||
if (numItems == 0)
|
||||
throw std::invalid_argument("Number of items must be greater than 0");
|
||||
const uint16_t nodeSizeMin = std::min(std::max(nodeSize, static_cast<uint16_t>(2)), static_cast<uint16_t>(65535));
|
||||
// limit so that resulting size in bytes can be represented by uint64_t
|
||||
if (numItems > static_cast<uint64_t>(1) << 56)
|
||||
throw std::overflow_error("Number of items must be less than 2^56");
|
||||
uint64_t n = numItems;
|
||||
uint64_t numNodes = n;
|
||||
do {
|
||||
n = (n + nodeSizeMin - 1) / nodeSizeMin;
|
||||
numNodes += n;
|
||||
} while (n != 1);
|
||||
return numNodes * sizeof(NodeItem);
|
||||
}
|
||||
|
||||
drawvec readPoints(const FlatGeobuf::Geometry *geometry) {
|
||||
auto xy = geometry->xy();
|
||||
drawvec dv;
|
||||
|
||||
for (unsigned int i = 0; i < xy->size(); i+=2) {
|
||||
for (unsigned int i = 0; i < xy->size(); i += 2) {
|
||||
long long x, y;
|
||||
projection->project(xy->Get(i), xy->Get(i+1), 32, &x, &y);
|
||||
projection->project(xy->Get(i), xy->Get(i + 1), 32, &x, &y);
|
||||
dv.push_back(draw(VT_MOVETO, x, y));
|
||||
}
|
||||
return dv;
|
||||
@@ -56,12 +55,13 @@ drawvec readLinePart(const FlatGeobuf::Geometry *geometry) {
|
||||
size_t current_end = 0;
|
||||
drawvec dv;
|
||||
|
||||
for (unsigned int i = 0; i < xy->size(); i+=2) {
|
||||
for (unsigned int i = 0; i < xy->size(); i += 2) {
|
||||
long long x, y;
|
||||
projection->project(xy->Get(i), xy->Get(i+1), 32, &x, &y);
|
||||
if (i == 0 || (ends != NULL && current_end < ends->size() && i == ends->Get(current_end)*2)) {
|
||||
projection->project(xy->Get(i), xy->Get(i + 1), 32, &x, &y);
|
||||
if (i == 0 || (ends != NULL && current_end < ends->size() && i == ends->Get(current_end) * 2)) {
|
||||
dv.push_back(draw(VT_MOVETO, x, y));
|
||||
if (i > 0) current_end++;
|
||||
if (i > 0)
|
||||
current_end++;
|
||||
} else {
|
||||
dv.push_back(draw(VT_LINETO, x, y));
|
||||
}
|
||||
@@ -71,20 +71,24 @@ drawvec readLinePart(const FlatGeobuf::Geometry *geometry) {
|
||||
|
||||
drawvec readGeometry(const FlatGeobuf::Geometry *geometry, FlatGeobuf::GeometryType h_geometry_type) {
|
||||
FlatGeobuf::GeometryType geometry_type = h_geometry_type;
|
||||
if (h_geometry_type == FlatGeobuf::GeometryType_Unknown) geometry_type = geometry->type();
|
||||
if (h_geometry_type == FlatGeobuf::GeometryType_Unknown)
|
||||
geometry_type = geometry->type();
|
||||
|
||||
if (geometry_type == FlatGeobuf::GeometryType_Point) {
|
||||
return readPoints(geometry);
|
||||
} if (geometry_type == FlatGeobuf::GeometryType_MultiPoint) {
|
||||
return readPoints(geometry);
|
||||
} if (geometry_type == FlatGeobuf::GeometryType_LineString) {
|
||||
}
|
||||
if (geometry_type == FlatGeobuf::GeometryType_MultiPoint) {
|
||||
return readPoints(geometry);
|
||||
}
|
||||
if (geometry_type == FlatGeobuf::GeometryType_LineString) {
|
||||
return readLinePart(geometry);
|
||||
} else if (h_geometry_type == FlatGeobuf::GeometryType_MultiLineString) {
|
||||
return readLinePart(geometry);
|
||||
} if (geometry_type == FlatGeobuf::GeometryType_Polygon) {
|
||||
}
|
||||
if (geometry_type == FlatGeobuf::GeometryType_Polygon) {
|
||||
return readLinePart(geometry);
|
||||
} else if (geometry_type == FlatGeobuf::GeometryType_MultiPolygon) {
|
||||
// if it is a GeometryCollection, parse Parts, ignore XY
|
||||
// if it is a GeometryCollection, parse Parts, ignore XY
|
||||
drawvec dv;
|
||||
for (size_t part = 0; part < geometry->parts()->size(); part++) {
|
||||
drawvec dv2 = readLinePart(geometry->parts()->Get(part));
|
||||
@@ -95,7 +99,7 @@ drawvec readGeometry(const FlatGeobuf::Geometry *geometry, FlatGeobuf::GeometryT
|
||||
}
|
||||
return dv;
|
||||
} else {
|
||||
fprintf(stderr, "flatgeobuf has unsupported geometry type %u\n", (unsigned int)h_geometry_type);
|
||||
fprintf(stderr, "flatgeobuf has unsupported geometry type %u\n", (unsigned int) h_geometry_type);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
@@ -106,26 +110,27 @@ void readFeature(const FlatGeobuf::Feature *feature, long long feature_sequence_
|
||||
int drawvec_type = -1;
|
||||
|
||||
FlatGeobuf::GeometryType geometry_type = h_geometry_type;
|
||||
if (h_geometry_type == FlatGeobuf::GeometryType_Unknown) geometry_type = feature->geometry()->type();
|
||||
if (h_geometry_type == FlatGeobuf::GeometryType_Unknown)
|
||||
geometry_type = feature->geometry()->type();
|
||||
|
||||
switch (geometry_type) {
|
||||
case FlatGeobuf::GeometryType_Point :
|
||||
case FlatGeobuf::GeometryType_MultiPoint :
|
||||
drawvec_type = 1;
|
||||
break;
|
||||
case FlatGeobuf::GeometryType_LineString :
|
||||
case FlatGeobuf::GeometryType_MultiLineString :
|
||||
drawvec_type = 2;
|
||||
break;
|
||||
case FlatGeobuf::GeometryType_Polygon :
|
||||
case FlatGeobuf::GeometryType_MultiPolygon :
|
||||
drawvec_type = 3;
|
||||
break;
|
||||
case FlatGeobuf::GeometryType_Unknown :
|
||||
case FlatGeobuf::GeometryType_GeometryCollection :
|
||||
default:
|
||||
fprintf(stderr, "flatgeobuf has unsupported geometry type %u\n", (unsigned int)h_geometry_type);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
case FlatGeobuf::GeometryType_Point:
|
||||
case FlatGeobuf::GeometryType_MultiPoint:
|
||||
drawvec_type = 1;
|
||||
break;
|
||||
case FlatGeobuf::GeometryType_LineString:
|
||||
case FlatGeobuf::GeometryType_MultiLineString:
|
||||
drawvec_type = 2;
|
||||
break;
|
||||
case FlatGeobuf::GeometryType_Polygon:
|
||||
case FlatGeobuf::GeometryType_MultiPolygon:
|
||||
drawvec_type = 3;
|
||||
break;
|
||||
case FlatGeobuf::GeometryType_Unknown:
|
||||
case FlatGeobuf::GeometryType_GeometryCollection:
|
||||
default:
|
||||
fprintf(stderr, "flatgeobuf has unsupported geometry type %u\n", (unsigned int) h_geometry_type);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
serial_feature sf;
|
||||
@@ -149,98 +154,117 @@ void readFeature(const FlatGeobuf::Feature *feature, long long feature_sequence_
|
||||
std::vector<std::string> full_keys;
|
||||
std::vector<serial_val> full_values;
|
||||
|
||||
// assume tabular schema with columns in header
|
||||
size_t p_pos = 0;
|
||||
while (p_pos < feature->properties()->size()) {
|
||||
uint16_t col_idx;
|
||||
memcpy(&col_idx, feature->properties()->data() + p_pos, sizeof(col_idx));
|
||||
if (feature == NULL) {
|
||||
fprintf(stderr, "Can't happen: null geobuf feature\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
if (feature->properties() != NULL) {
|
||||
size_t p_pos = 0;
|
||||
while (p_pos < feature->properties()->size()) {
|
||||
uint16_t col_idx;
|
||||
memcpy(&col_idx, feature->properties()->data() + p_pos, sizeof(col_idx));
|
||||
|
||||
// https://github.com/protomaps/tippecanoe/issues/7
|
||||
// check if column name is tippecanoe:minzoom, tippecanoe:maxzoom or tippecanoe:layer
|
||||
// https://github.com/protomaps/tippecanoe/issues/7
|
||||
// check if column name is tippecanoe:minzoom, tippecanoe:maxzoom or tippecanoe:layer
|
||||
|
||||
FlatGeobuf::ColumnType col_type = h_column_types[col_idx];
|
||||
FlatGeobuf::ColumnType col_type;
|
||||
if (feature->columns() != NULL) {
|
||||
col_type = feature->columns()->Get(col_idx)->type();
|
||||
} else {
|
||||
col_type = h_column_types[col_idx];
|
||||
}
|
||||
|
||||
serial_val sv;
|
||||
if (col_type == FlatGeobuf::ColumnType_Byte) {
|
||||
sv.type = mvt_sint;
|
||||
int8_t byte_val;
|
||||
memcpy(&byte_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(byte_val));
|
||||
sv.s = std::to_string(byte_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(byte_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_UByte) {
|
||||
sv.type = mvt_uint;
|
||||
uint8_t ubyte_val;
|
||||
memcpy(&ubyte_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(ubyte_val));
|
||||
sv.s = std::to_string(ubyte_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(ubyte_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Bool) {
|
||||
sv.type = mvt_bool;
|
||||
uint8_t bool_val;
|
||||
memcpy(&bool_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(bool_val));
|
||||
sv.s = std::to_string(bool_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(bool_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Short) {
|
||||
sv.type = mvt_sint;
|
||||
int16_t short_val;
|
||||
memcpy(&short_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(short_val));
|
||||
sv.s = std::to_string(short_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(short_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_UShort) {
|
||||
sv.type = mvt_uint;
|
||||
uint16_t ushort_val;
|
||||
memcpy(&ushort_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(ushort_val));
|
||||
sv.s = std::to_string(ushort_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(ushort_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Int) {
|
||||
sv.type = mvt_sint;
|
||||
int32_t int_val;
|
||||
memcpy(&int_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(int_val));
|
||||
sv.s = std::to_string(int_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(int_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_UInt) {
|
||||
sv.type = mvt_uint;
|
||||
uint32_t uint_val;
|
||||
memcpy(&uint_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(uint_val));
|
||||
sv.s = std::to_string(uint_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(uint_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Long) {
|
||||
sv.type = mvt_sint;
|
||||
int64_t long_val;
|
||||
memcpy(&long_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(long_val));
|
||||
sv.s = std::to_string(long_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(long_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_ULong) {
|
||||
sv.type = mvt_uint;
|
||||
int64_t ulong_val;
|
||||
memcpy(&ulong_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(ulong_val));
|
||||
sv.s = std::to_string(ulong_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(ulong_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Float) {
|
||||
sv.type = mvt_float;
|
||||
float float_val;
|
||||
memcpy(&float_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(float_val));
|
||||
sv.s = milo::dtoa_milo(float_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(float_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Double) {
|
||||
sv.type = mvt_double;
|
||||
double double_val;
|
||||
memcpy(&double_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(double_val));
|
||||
sv.s = milo::dtoa_milo(double_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(double_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_String || col_type == FlatGeobuf::ColumnType_Json || col_type == FlatGeobuf::ColumnType_DateTime) {
|
||||
sv.type = mvt_string;
|
||||
uint32_t val_len;
|
||||
memcpy(&val_len, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(val_len));
|
||||
std::string s{reinterpret_cast<const char*>(feature->properties()->data() + p_pos + sizeof(uint16_t) + sizeof(uint32_t)), val_len};
|
||||
sv.s = s;
|
||||
p_pos += sizeof(uint16_t) + sizeof(uint32_t) + val_len;
|
||||
} else {
|
||||
// Binary is not representable in MVT
|
||||
fprintf(stderr, "flatgeobuf has unsupported column type %u\n", (unsigned int)col_type);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
serial_val sv;
|
||||
// All of the numeric types are tagged as mvt_double rather than with their true type
|
||||
// because tippecanoe expects values in the string pool to be tagged that way. Sorry!
|
||||
if (col_type == FlatGeobuf::ColumnType_Byte) {
|
||||
sv.type = mvt_double;
|
||||
int8_t byte_val;
|
||||
memcpy(&byte_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(byte_val));
|
||||
sv.s = std::to_string(byte_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(byte_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_UByte) {
|
||||
sv.type = mvt_double;
|
||||
uint8_t ubyte_val;
|
||||
memcpy(&ubyte_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(ubyte_val));
|
||||
sv.s = std::to_string(ubyte_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(ubyte_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Bool) {
|
||||
sv.type = mvt_bool;
|
||||
uint8_t bool_val;
|
||||
memcpy(&bool_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(bool_val));
|
||||
sv.s = bool_val ? "true" : "false";
|
||||
sv.type = mvt_string;
|
||||
p_pos += sizeof(uint16_t) + sizeof(bool_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Short) {
|
||||
sv.type = mvt_double;
|
||||
int16_t short_val;
|
||||
memcpy(&short_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(short_val));
|
||||
sv.s = std::to_string(short_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(short_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_UShort) {
|
||||
sv.type = mvt_double;
|
||||
uint16_t ushort_val;
|
||||
memcpy(&ushort_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(ushort_val));
|
||||
sv.s = std::to_string(ushort_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(ushort_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Int) {
|
||||
sv.type = mvt_double;
|
||||
int32_t int_val;
|
||||
memcpy(&int_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(int_val));
|
||||
sv.s = std::to_string(int_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(int_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_UInt) {
|
||||
sv.type = mvt_double;
|
||||
uint32_t uint_val;
|
||||
memcpy(&uint_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(uint_val));
|
||||
sv.s = std::to_string(uint_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(uint_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Long) {
|
||||
sv.type = mvt_double;
|
||||
int64_t long_val;
|
||||
memcpy(&long_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(long_val));
|
||||
sv.s = std::to_string(long_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(long_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_ULong) {
|
||||
sv.type = mvt_double;
|
||||
int64_t ulong_val;
|
||||
memcpy(&ulong_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(ulong_val));
|
||||
sv.s = std::to_string(ulong_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(ulong_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Float) {
|
||||
sv.type = mvt_double;
|
||||
float float_val;
|
||||
memcpy(&float_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(float_val));
|
||||
sv.s = milo::dtoa_milo(float_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(float_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_Double) {
|
||||
sv.type = mvt_double;
|
||||
double double_val;
|
||||
memcpy(&double_val, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(double_val));
|
||||
sv.s = milo::dtoa_milo(double_val);
|
||||
p_pos += sizeof(uint16_t) + sizeof(double_val);
|
||||
} else if (col_type == FlatGeobuf::ColumnType_String || col_type == FlatGeobuf::ColumnType_Json || col_type == FlatGeobuf::ColumnType_DateTime) {
|
||||
sv.type = mvt_string;
|
||||
uint32_t val_len;
|
||||
memcpy(&val_len, feature->properties()->data() + p_pos + sizeof(uint16_t), sizeof(val_len));
|
||||
std::string s{reinterpret_cast<const char *>(feature->properties()->data() + p_pos + sizeof(uint16_t) + sizeof(uint32_t)), val_len};
|
||||
sv.s = s;
|
||||
p_pos += sizeof(uint16_t) + sizeof(uint32_t) + val_len;
|
||||
} else {
|
||||
// Binary is not representable in MVT
|
||||
fprintf(stderr, "flatgeobuf has unsupported column type %u\n", (unsigned int) col_type);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
if (feature->columns() != NULL) {
|
||||
full_keys.push_back(feature->columns()->Get(col_idx)->name()->c_str());
|
||||
} else {
|
||||
full_keys.push_back(h_column_names[col_idx]);
|
||||
}
|
||||
|
||||
full_values.push_back(sv);
|
||||
}
|
||||
full_keys.push_back(h_column_names[col_idx]);
|
||||
full_values.push_back(sv);
|
||||
}
|
||||
|
||||
sf.full_keys = full_keys;
|
||||
@@ -342,9 +366,9 @@ void queueFeature(const FlatGeobuf::Feature *feature, long long feature_sequence
|
||||
}
|
||||
|
||||
void parse_flatgeobuf(std::vector<struct serialization_state> *sst, const char *src, size_t len, int layer, std::string layername) {
|
||||
auto header_size = flatbuffers::GetPrefixedSize((const uint8_t *)src + sizeof(magicbytes));
|
||||
auto header_size = flatbuffers::GetPrefixedSize((const uint8_t *) src + sizeof(magicbytes));
|
||||
|
||||
flatbuffers::Verifier v((const uint8_t *)src+sizeof(magicbytes),header_size+sizeof(uint32_t));
|
||||
flatbuffers::Verifier v((const uint8_t *) src + sizeof(magicbytes), header_size + sizeof(uint32_t));
|
||||
const auto ok = FlatGeobuf::VerifySizePrefixedHeaderBuffer(v);
|
||||
if (!ok) {
|
||||
fprintf(stderr, "flatgeobuf header verification failed\n");
|
||||
@@ -370,18 +394,14 @@ void parse_flatgeobuf(std::vector<struct serialization_state> *sst, const char *
|
||||
long long feature_sequence_id = -1;
|
||||
int index_size = 0;
|
||||
if (node_size > 0) {
|
||||
if (!quiet) {
|
||||
fprintf(stderr, "detected indexed FlatGeobuf: assigning feature IDs by sequence\n");
|
||||
}
|
||||
index_size = PackedRTreeSize(features_count,node_size);
|
||||
feature_sequence_id = 0;
|
||||
index_size = PackedRTreeSize(features_count, node_size);
|
||||
}
|
||||
const char* start = src + sizeof(magicbytes) + sizeof(uint32_t) + header_size + index_size;
|
||||
const char *start = src + sizeof(magicbytes) + sizeof(uint32_t) + header_size + index_size;
|
||||
|
||||
while (start < src + len) {
|
||||
auto feature_size = flatbuffers::GetPrefixedSize((const uint8_t *)start);
|
||||
auto feature_size = flatbuffers::GetPrefixedSize((const uint8_t *) start);
|
||||
|
||||
flatbuffers::Verifier v2((const uint8_t *)start,feature_size+sizeof(uint32_t));
|
||||
flatbuffers::Verifier v2((const uint8_t *) start, feature_size + sizeof(uint32_t));
|
||||
const auto ok2 = FlatGeobuf::VerifySizePrefixedFeatureBuffer(v2);
|
||||
if (!ok2) {
|
||||
fprintf(stderr, "flatgeobuf feature buffer verification failed\n");
|
||||
@@ -392,7 +412,6 @@ void parse_flatgeobuf(std::vector<struct serialization_state> *sst, const char *
|
||||
|
||||
queueFeature(feature, feature_sequence_id, h_geometry_type, h_column_names, h_column_types, sst, layer, layername);
|
||||
|
||||
if (feature_sequence_id >= 0) feature_sequence_id ++;
|
||||
start += sizeof(uint32_t) + feature_size;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user