diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-06 02:15:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update flatbuffers to v22.9.29 (#26)

Browse Source 
* update flatbuffers files

How:
- Copied files in from the latest release (v22.9.29) of flatbuffers

Why:
- The files in place blocked compilation within Alpine Linux

Links:
- https://github.com/felt/tippecanoe/issues/24
- https://github.com/google/flatbuffers/releases/tag/v22.9.29

* update flatgeobuf generate files

How
- Created `feature_generated.h` & `header_generated.h` using the `src/fbs` files from the latest release (3.23.1) of flatgeobuf
    - Downloaded the `flatc` binary from the v22.9.29 release of flatbuffers
    - Copied `flatc` into the flatgeobuf repository
    - Created a new directory named `out` to collect the generated files
    - `./flatc --cpp -o out -I include src/fbs/feature.fbs`
    - `./flatc --cpp -o out -I include src/fbs/header.fbs`
    - Moved the generate files from `out` into this repository's `flatgeobuf` directory

Why
- This is a required step in the process of updating flatbuffers files

Links
- https://github.com/flatgeobuf/flatgeobuf/releases/tag/3.23.1
- https://github.com/google/flatbuffers/releases/tag/v22.9.29
- https://github.com/felt/tippecanoe/issues/24#issuecomment-1286476550
This commit is contained in:
James Chevalier
2022-11-03 14:24:02 -07:00
committed by GitHub
parent 5b03b18f31
commit c2214631f2
12 changed files with 436 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flatbuffers/verifier.h
+114 -67
View File
@@ -18,7 +18,6 @@
#define FLATBUFFERS_VERIFIER_H_
#include "flatbuffers/base.h"
#include "flatbuffers/util.h"
#include "flatbuffers/vector.h"
namespace flatbuffers {
@@ -26,21 +25,37 @@ namespace flatbuffers {
// Helper class to verify the integrity of a FlatBuffer
class Verifier FLATBUFFERS_FINAL_CLASS {
public:
Verifier(const uint8_t *buf, size_t buf_len, uoffset_t _max_depth = 64,
uoffset_t _max_tables = 1000000, bool _check_alignment = true)
: buf_(buf),
size_(buf_len),
depth_(0),
max_depth_(_max_depth),
num_tables_(0),
max_tables_(_max_tables),
upper_bound_(0),
check_alignment_(_check_alignment) {
struct Options {
// The maximum nesting of tables and vectors before we call it invalid.
uoffset_t max_depth = 64;
// The maximum number of tables we will verify before we call it invalid.
uoffset_t max_tables = 1000000;
// If true, verify all data is aligned.
bool check_alignment = true;
// If true, run verifier on nested flatbuffers
bool check_nested_flatbuffers = true;
};
explicit Verifier(const uint8_t *const buf, const size_t buf_len,
const Options &opts)
: buf_(buf), size_(buf_len), opts_(opts) {
FLATBUFFERS_ASSERT(size_ < FLATBUFFERS_MAX_BUFFER_SIZE);
}
// Deprecated API, please construct with Verifier::Options.
Verifier(const uint8_t *const buf, const size_t buf_len,
const uoffset_t max_depth = 64, const uoffset_t max_tables = 1000000,
const bool check_alignment = true)
: Verifier(buf, buf_len, [&] {
Options opts;
opts.max_depth = max_depth;
opts.max_tables = max_tables;
opts.check_alignment = check_alignment;
return opts;
}()) {}
// Central location where any verification failures register.
bool Check(bool ok) const {
bool Check(const bool ok) const {
// clang-format off
#ifdef FLATBUFFERS_DEBUG_VERIFICATION_FAILURE
FLATBUFFERS_ASSERT(ok);
@@ -54,7 +69,7 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
}
// Verify any range within the buffer.
bool Verify(size_t elem, size_t elem_len) const {
bool Verify(const size_t elem, const size_t elem_len) const {
// clang-format off
#ifdef FLATBUFFERS_TRACK_VERIFIER_BUFFER_SIZE
auto upper_bound = elem + elem_len;
@@ -65,48 +80,52 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
return Check(elem_len < size_ && elem <= size_ - elem_len);
}
template<typename T> bool VerifyAlignment(size_t elem) const {
return Check((elem & (sizeof(T) - 1)) == 0 || !check_alignment_);
bool VerifyAlignment(const size_t elem, const size_t align) const {
return Check((elem & (align - 1)) == 0 || !opts_.check_alignment);
}
// Verify a range indicated by sizeof(T).
template<typename T> bool Verify(size_t elem) const {
return VerifyAlignment<T>(elem) && Verify(elem, sizeof(T));
template<typename T> bool Verify(const size_t elem) const {
return VerifyAlignment(elem, sizeof(T)) && Verify(elem, sizeof(T));
}
bool VerifyFromPointer(const uint8_t *p, size_t len) {
auto o = static_cast<size_t>(p - buf_);
return Verify(o, len);
bool VerifyFromPointer(const uint8_t *const p, const size_t len) {
return Verify(static_cast<size_t>(p - buf_), len);
}
// Verify relative to a known-good base pointer.
bool Verify(const uint8_t *base, voffset_t elem_off, size_t elem_len) const {
return Verify(static_cast<size_t>(base - buf_) + elem_off, elem_len);
bool VerifyFieldStruct(const uint8_t *const base, const voffset_t elem_off,
const size_t elem_len, const size_t align) const {
const auto f = static_cast<size_t>(base - buf_) + elem_off;
return VerifyAlignment(f, align) && Verify(f, elem_len);
}
template<typename T>
bool Verify(const uint8_t *base, voffset_t elem_off) const {
return Verify(static_cast<size_t>(base - buf_) + elem_off, sizeof(T));
bool VerifyField(const uint8_t *const base, const voffset_t elem_off,
const size_t align) const {
const auto f = static_cast<size_t>(base - buf_) + elem_off;
return VerifyAlignment(f, align) && Verify(f, sizeof(T));
}
// Verify a pointer (may be NULL) of a table type.
template<typename T> bool VerifyTable(const T *table) {
template<typename T> bool VerifyTable(const T *const table) {
return !table || table->Verify(*this);
}
// Verify a pointer (may be NULL) of any vector type.
template<typename T> bool VerifyVector(const Vector<T> *vec) const {
template<typename T> bool VerifyVector(const Vector<T> *const vec) const {
return !vec || VerifyVectorOrString(reinterpret_cast<const uint8_t *>(vec),
sizeof(T));
}
// Verify a pointer (may be NULL) of a vector to struct.
template<typename T> bool VerifyVector(const Vector<const T *> *vec) const {
template<typename T>
bool VerifyVector(const Vector<const T *> *const vec) const {
return VerifyVector(reinterpret_cast<const Vector<T> *>(vec));
}
// Verify a pointer (may be NULL) to string.
bool VerifyString(const String *str) const {
bool VerifyString(const String *const str) const {
size_t end;
return !str || (VerifyVectorOrString(reinterpret_cast<const uint8_t *>(str),
1, &end) &&
@@ -115,24 +134,24 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
}
// Common code between vectors and strings.
bool VerifyVectorOrString(const uint8_t *vec, size_t elem_size,
size_t *end = nullptr) const {
auto veco = static_cast<size_t>(vec - buf_);
bool VerifyVectorOrString(const uint8_t *const vec, const size_t elem_size,
size_t *const end = nullptr) const {
const auto veco = static_cast<size_t>(vec - buf_);
// Check we can read the size field.
if (!Verify<uoffset_t>(veco)) return false;
// Check the whole array. If this is a string, the byte past the array
// must be 0.
auto size = ReadScalar<uoffset_t>(vec);
auto max_elems = FLATBUFFERS_MAX_BUFFER_SIZE / elem_size;
// Check the whole array. If this is a string, the byte past the array must
// be 0.
const auto size = ReadScalar<uoffset_t>(vec);
const auto max_elems = FLATBUFFERS_MAX_BUFFER_SIZE / elem_size;
if (!Check(size < max_elems))
return false; // Protect against byte_size overflowing.
auto byte_size = sizeof(size) + elem_size * size;
const auto byte_size = sizeof(size) + elem_size * size;
if (end) *end = veco + byte_size;
return Verify(veco, byte_size);
}
// Special case for string contents, after the above has been called.
bool VerifyVectorOfStrings(const Vector<Offset<String>> *vec) const {
bool VerifyVectorOfStrings(const Vector<Offset<String>> *const vec) const {
if (vec) {
for (uoffset_t i = 0; i < vec->size(); i++) {
if (!VerifyString(vec->Get(i))) return false;
@@ -142,7 +161,8 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
}
// Special case for table contents, after the above has been called.
template<typename T> bool VerifyVectorOfTables(const Vector<Offset<T>> *vec) {
template<typename T>
bool VerifyVectorOfTables(const Vector<Offset<T>> *const vec) {
if (vec) {
for (uoffset_t i = 0; i < vec->size(); i++) {
if (!vec->Get(i)->Verify(*this)) return false;
@@ -151,30 +171,41 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
return true;
}
__supress_ubsan__("unsigned-integer-overflow") bool VerifyTableStart(
const uint8_t *table) {
__suppress_ubsan__("unsigned-integer-overflow") bool VerifyTableStart(
const uint8_t *const table) {
// Check the vtable offset.
auto tableo = static_cast<size_t>(table - buf_);
const auto tableo = static_cast<size_t>(table - buf_);
if (!Verify<soffset_t>(tableo)) return false;
// This offset may be signed, but doing the subtraction unsigned always
// gives the result we want.
auto vtableo = tableo - static_cast<size_t>(ReadScalar<soffset_t>(table));
const auto vtableo =
tableo - static_cast<size_t>(ReadScalar<soffset_t>(table));
// Check the vtable size field, then check vtable fits in its entirety.
return VerifyComplexity() && Verify<voffset_t>(vtableo) &&
VerifyAlignment<voffset_t>(ReadScalar<voffset_t>(buf_ + vtableo)) &&
Verify(vtableo, ReadScalar<voffset_t>(buf_ + vtableo));
if (!(VerifyComplexity() && Verify<voffset_t>(vtableo) &&
VerifyAlignment(ReadScalar<voffset_t>(buf_ + vtableo),
sizeof(voffset_t))))
return false;
const auto vsize = ReadScalar<voffset_t>(buf_ + vtableo);
return Check((vsize & 1) == 0) && Verify(vtableo, vsize);
}
template<typename T>
bool VerifyBufferFromStart(const char *identifier, size_t start) {
bool VerifyBufferFromStart(const char *const identifier, const size_t start) {
// Buffers have to be of some size to be valid. The reason it is a runtime
// check instead of static_assert, is that nested flatbuffers go through
// this call and their size is determined at runtime.
if (!Check(size_ >= FLATBUFFERS_MIN_BUFFER_SIZE)) return false;
// If an identifier is provided, check that we have a buffer
if (identifier && !Check((size_ >= 2 * sizeof(flatbuffers::uoffset_t) &&
BufferHasIdentifier(buf_ + start, identifier)))) {
return false;
}
// Call T::Verify, which must be in the generated code for this type.
auto o = VerifyOffset(start);
return o && reinterpret_cast<const T *>(buf_ + start + o)->Verify(*this)
const auto o = VerifyOffset(start);
return Check(o != 0) &&
reinterpret_cast<const T *>(buf_ + start + o)->Verify(*this)
// clang-format off
#ifdef FLATBUFFERS_TRACK_VERIFIER_BUFFER_SIZE
&& GetComputedSize()
@@ -184,9 +215,17 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
}
template<typename T>
bool VerifyNestedFlatBuffer(const Vector<uint8_t> *buf,
const char *identifier) {
bool VerifyNestedFlatBuffer(const Vector<uint8_t> *const buf,
const char *const identifier) {
// Caller opted out of this.
if (!opts_.check_nested_flatbuffers) return true;
// An empty buffer is OK as it indicates not present.
if (!buf) return true;
// If there is a nested buffer, it must be greater than the min size.
if (!Check(buf->size() >= FLATBUFFERS_MIN_BUFFER_SIZE)) return false;
Verifier nested_verifier(buf->data(), buf->size());
return nested_verifier.VerifyBuffer<T>(identifier);
}
@@ -194,19 +233,20 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
// Verify this whole buffer, starting with root type T.
template<typename T> bool VerifyBuffer() { return VerifyBuffer<T>(nullptr); }
template<typename T> bool VerifyBuffer(const char *identifier) {
template<typename T> bool VerifyBuffer(const char *const identifier) {
return VerifyBufferFromStart<T>(identifier, 0);
}
template<typename T> bool VerifySizePrefixedBuffer(const char *identifier) {
template<typename T>
bool VerifySizePrefixedBuffer(const char *const identifier) {
return Verify<uoffset_t>(0U) &&
ReadScalar<uoffset_t>(buf_) == size_ - sizeof(uoffset_t) &&
Check(ReadScalar<uoffset_t>(buf_) == size_ - sizeof(uoffset_t)) &&
VerifyBufferFromStart<T>(identifier, sizeof(uoffset_t));
}
uoffset_t VerifyOffset(size_t start) const {
uoffset_t VerifyOffset(const size_t start) const {
if (!Verify<uoffset_t>(start)) return 0;
auto o = ReadScalar<uoffset_t>(buf_ + start);
const auto o = ReadScalar<uoffset_t>(buf_ + start);
// May not point to itself.
if (!Check(o != 0)) return 0;
// Can't wrap around / buffers are max 2GB.
@@ -217,18 +257,18 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
return o;
}
uoffset_t VerifyOffset(const uint8_t *base, voffset_t start) const {
uoffset_t VerifyOffset(const uint8_t *const base,
const voffset_t start) const {
return VerifyOffset(static_cast<size_t>(base - buf_) + start);
}
// Called at the start of a table to increase counters measuring data
// structure depth and amount, and possibly bails out with false if
// limits set by the constructor have been hit. Needs to be balanced
// with EndTable().
// structure depth and amount, and possibly bails out with false if limits set
// by the constructor have been hit. Needs to be balanced with EndTable().
bool VerifyComplexity() {
depth_++;
num_tables_++;
return Check(depth_ <= max_depth_ && num_tables_ <= max_tables_);
return Check(depth_ <= opts_.max_depth && num_tables_ <= opts_.max_tables);
}
// Called at the end of a table to pop the depth count.
@@ -254,15 +294,22 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
// clang-format on
}
std::vector<uint8_t> *GetFlexReuseTracker() { return flex_reuse_tracker_; }
void SetFlexReuseTracker(std::vector<uint8_t> *const rt) {
flex_reuse_tracker_ = rt;
}
private:
const uint8_t *buf_;
size_t size_;
uoffset_t depth_;
uoffset_t max_depth_;
uoffset_t num_tables_;
uoffset_t max_tables_;
mutable size_t upper_bound_;
bool check_alignment_;
const size_t size_;
const Options opts_;
mutable size_t upper_bound_ = 0;
uoffset_t depth_ = 0;
uoffset_t num_tables_ = 0;
std::vector<uint8_t> *flex_reuse_tracker_ = nullptr;
};
} // namespace flatbuffers