diff --git a/Makefile b/Makefile index 7d3a4f40..64657843 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ PG= H = $(shell find . '(' -name '*.h' -o -name '*.hh' ')') C = $(shell find . '(' -name '*.c' -o -name '*.cc' ')') -INCLUDES = -I/usr/local/include +INCLUDES = -I/usr/local/include -I. LIBS = -L/usr/local/lib tippecanoe: geojson.o jsonpull.o vector_tile.pb.o tile.o clip.o pool.o mbtiles.o geometry.o projection.o memfile.o clipper/clipper.o mvt.o diff --git a/mvt.cc b/mvt.cc index 426eebbf..a257018f 100644 --- a/mvt.cc +++ b/mvt.cc @@ -6,6 +6,7 @@ #include #include "mvt.hh" #include "vector_tile.pb.h" +#include "protozero/pbf_reader.hpp" // https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp bool is_compressed(std::string const &data) { @@ -76,97 +77,160 @@ int dezig(unsigned n) { } bool mvt_decode(std::string &message, mvt_tile &out) { - GOOGLE_PROTOBUF_VERIFY_VERSION; - - // https://github.com/mapbox/mapnik-vector-tile/blob/master/examples/c%2B%2B/tileinfo.cpp - mapnik::vector::tile tile; out.layers.clear(); + std::string src; if (is_compressed(message)) { std::string uncompressed; decompress(message, uncompressed); - google::protobuf::io::ArrayInputStream stream(uncompressed.c_str(), uncompressed.length()); - google::protobuf::io::CodedInputStream codedstream(&stream); - codedstream.SetTotalBytesLimit(10 * 67108864, 5 * 67108864); - if (!tile.ParseFromCodedStream(&codedstream)) { - return false; - } - } else if (!tile.ParseFromString(message)) { - return false; + src = uncompressed; + } else { + src = message; } - for (size_t l = 0; l < tile.layers_size(); l++) { - mapnik::vector::tile_layer layer = tile.layers(l); - mvt_layer pbl; - pbl.extent = layer.extent(); - pbl.name = layer.name(); + protozero::pbf_reader reader(src); - for (size_t i = 0; i < layer.keys_size(); i++) { - pbl.keys.push_back(layer.keys(i)); - } + while (reader.next()) { + switch (reader.tag()) { + case 3: /* layer */ + { + protozero::pbf_reader layer_reader(reader.get_message()); + mvt_layer layer; - for (size_t i = 0; i < layer.values_size(); i++) { - mapnik::vector::tile_value const &val = layer.values(i); - mvt_value pbv; + while (layer_reader.next()) { + switch (layer_reader.tag()) { + case 1: /* name */ + layer.name = layer_reader.get_string(); + break; - if (val.has_string_value()) { - pbv.type = mvt_string; - pbv.string_value = val.string_value(); - } else if (val.has_float_value()) { - pbv.type = mvt_float; - pbv.numeric_value.float_value = val.float_value(); - } else if (val.has_double_value()) { - pbv.type = mvt_double; - pbv.numeric_value.double_value = val.double_value(); - } else if (val.has_int_value()) { - pbv.type = mvt_int; - pbv.numeric_value.int_value = val.int_value(); - } else if (val.has_uint_value()) { - pbv.type = mvt_uint; - pbv.numeric_value.uint_value = val.uint_value(); - } else if (val.has_sint_value()) { - pbv.type = mvt_sint; - pbv.numeric_value.sint_value = val.sint_value(); - } else if (val.has_bool_value()) { - pbv.type = mvt_bool; - pbv.numeric_value.bool_value = val.bool_value(); - } + case 3: /* key */ + layer.keys.push_back(layer_reader.get_string()); + break; - pbl.values.push_back(pbv); - } + case 4: /* value */ + { + protozero::pbf_reader value_reader(layer_reader.get_message()); + mvt_value value; - for (size_t f = 0; f < layer.features_size(); f++) { - mapnik::vector::tile_feature feat = layer.features(f); - mvt_feature pbf; - pbf.type = feat.type(); + while (value_reader.next()) { + switch (value_reader.tag()) { + case 1: /* string */ + value.type = mvt_string; + value.string_value = value_reader.get_string(); + break; - for (size_t i = 0; i < feat.tags_size(); i++) { - pbf.tags.push_back(feat.tags(i)); - } + case 2: /* float */ + value.type = mvt_float; + value.numeric_value.float_value = value_reader.get_float(); + break; - long long px = 0, py = 0; - for (size_t g = 0; g < feat.geometry_size(); g++) { - uint32_t geom = feat.geometry(g); - uint32_t op = geom & 7; - uint32_t count = geom >> 3; + case 3: /* double */ + value.type = mvt_double; + value.numeric_value.double_value = value_reader.get_double(); + break; - if (op == mvt_moveto || op == mvt_lineto) { - for (size_t k = 0; k < count; k++) { - px += dezig(feat.geometry(g + 1)); - py += dezig(feat.geometry(g + 2)); - g += 2; + case 4: /* int */ + value.type = mvt_int; + value.numeric_value.int_value = value_reader.get_int64(); + break; - pbf.geometry.push_back(mvt_geometry(op, px, py)); + case 5: /* uint */ + value.type = mvt_uint; + value.numeric_value.uint_value = value_reader.get_uint64(); + break; + + case 6: /* sint */ + value.type = mvt_sint; + value.numeric_value.sint_value = value_reader.get_sint64(); + break; + + case 7: /* bool */ + value.type = mvt_bool; + value.numeric_value.bool_value = value_reader.get_bool(); + break; + + default: + value_reader.skip(); + break; + } } - } else { - pbf.geometry.push_back(mvt_geometry(op, 0, 0)); + + layer.values.push_back(value); + } break; + + case 5: /* extent */ + layer.extent = layer_reader.get_uint32(); + break; + + case 2: /* feature */ + { + protozero::pbf_reader feature_reader(layer_reader.get_message()); + mvt_feature feature; + std::vector geoms; + + while (feature_reader.next()) { + switch (feature_reader.tag()) { + case 2: /* tag */ + { + auto pi = feature_reader.get_packed_uint32(); + for (auto it = pi.first; it != pi.second; ++it) { + feature.tags.push_back(*it); + } + } break; + + case 3: /* feature type */ + feature.type = feature_reader.get_enum(); + break; + + case 4: /* geometry */ + { + auto pi = feature_reader.get_packed_uint32(); + for (auto it = pi.first; it != pi.second; ++it) { + geoms.push_back(*it); + } + } break; + + default: + feature_reader.skip(); + break; + } + } + + long long px = 0, py = 0; + for (size_t g = 0; g < geoms.size(); g++) { + uint32_t geom = geoms[g]; + uint32_t op = geom & 7; + uint32_t count = geom >> 3; + + if (op == mvt_moveto || op == mvt_lineto) { + for (size_t k = 0; k < count && g + 2 < geoms.size(); k++) { + px += dezig(geoms[g + 1]); + py += dezig(geoms[g + 2]); + g += 2; + + feature.geometry.push_back(mvt_geometry(op, px, py)); + } + } else { + feature.geometry.push_back(mvt_geometry(op, 0, 0)); + } + } + + layer.features.push_back(feature); + } break; + + default: + layer_reader.skip(); + break; } } - pbl.features.push_back(pbf); - } + out.layers.push_back(layer); + } break; - out.layers.push_back(pbl); + default: + reader.skip(); + break; + } } return true; diff --git a/mvt.hh b/mvt.hh index a92fa5ce..44515357 100644 --- a/mvt.hh +++ b/mvt.hh @@ -4,15 +4,25 @@ int compress(std::string const &input, std::string &output); int dezig(unsigned n); enum mvt_geometry_type { - mvt_point = 1, mvt_linestring = 2, mvt_polygon = 3 + mvt_point = 1, + mvt_linestring = 2, + mvt_polygon = 3 }; enum mvt_operation { - mvt_moveto = 1, mvt_lineto = 2, mvt_closepath = 7 + mvt_moveto = 1, + mvt_lineto = 2, + mvt_closepath = 7 }; enum mvt_value_type { - mvt_string, mvt_float, mvt_double, mvt_int, mvt_uint, mvt_sint, mvt_bool + mvt_string, + mvt_float, + mvt_double, + mvt_int, + mvt_uint, + mvt_sint, + mvt_bool }; struct mvt_value { diff --git a/protozero/byteswap.hpp b/protozero/byteswap.hpp new file mode 100644 index 00000000..a018c1c1 --- /dev/null +++ b/protozero/byteswap.hpp @@ -0,0 +1,71 @@ +#ifndef PROTOZERO_BYTESWAP_HPP +#define PROTOZERO_BYTESWAP_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file byteswap.hpp + * + * @brief Contains functions to swap bytes in values (for different endianness). + */ + +#include +#include + +#include + +namespace protozero { + +/** + * Swap N byte value between endianness formats. This template function must + * be specialized to actually work. + */ +template +inline void byteswap(const char* /*data*/, char* /*result*/) { + static_assert(N == 1, "Can only swap 4 or 8 byte values"); +} + +/** + * Swap 4 byte value (int32_t, uint32_t, float) between endianness formats. + */ +template <> +inline void byteswap<4>(const char* data, char* result) { +#ifdef PROTOZERO_USE_BUILTIN_BSWAP + *reinterpret_cast(result) = __builtin_bswap32(*reinterpret_cast(data)); +#else + result[3] = data[0]; + result[2] = data[1]; + result[1] = data[2]; + result[0] = data[3]; +#endif +} + +/** + * Swap 8 byte value (int64_t, uint64_t, double) between endianness formats. + */ +template <> +inline void byteswap<8>(const char* data, char* result) { +#ifdef PROTOZERO_USE_BUILTIN_BSWAP + *reinterpret_cast(result) = __builtin_bswap64(*reinterpret_cast(data)); +#else + result[7] = data[0]; + result[6] = data[1]; + result[5] = data[2]; + result[4] = data[3]; + result[3] = data[4]; + result[2] = data[5]; + result[1] = data[6]; + result[0] = data[7]; +#endif +} + +} // end namespace protozero + +#endif // PROTOZERO_BYTESWAP_HPP diff --git a/protozero/config.hpp b/protozero/config.hpp new file mode 100644 index 00000000..8465c968 --- /dev/null +++ b/protozero/config.hpp @@ -0,0 +1,59 @@ +#ifndef PROTOZERO_CONFIG_HPP +#define PROTOZERO_CONFIG_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +/** + * @file config.hpp + * + * @brief Contains macro checks for different configurations. + */ + +#define PROTOZERO_LITTLE_ENDIAN 1234 +#define PROTOZERO_BIG_ENDIAN 4321 + +// Find out which byte order the machine has. +#if defined(__BYTE_ORDER) +# if (__BYTE_ORDER == __LITTLE_ENDIAN) +# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN +# endif +# if (__BYTE_ORDER == __BIG_ENDIAN) +# define PROTOZERO_BYTE_ORDER PROTOZERO_BIG_ENDIAN +# endif +#else +// This probably isn't a very good default, but might do until we figure +// out something better. +# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN +#endif + +// On some ARM machines and depending on compiler settings access to unaligned +// floating point values will result in a SIGBUS. Do not use the bare pointers +// in this case. +#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN +# if !defined(__arm__) && !defined(_M_ARM) +# ifndef PROTOZERO_DO_NOT_USE_BARE_POINTER +# define PROTOZERO_USE_BARE_POINTER_FOR_PACKED_FIXED +# endif +# endif +#endif + +// Check whether __builtin_bswap is available +#if defined(__GNUC__) || defined(__clang__) +# define PROTOZERO_USE_BUILTIN_BSWAP +#endif + +// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +#endif // PROTOZERO_CONFIG_HPP diff --git a/protozero/exception.hpp b/protozero/exception.hpp new file mode 100644 index 00000000..5c7ab547 --- /dev/null +++ b/protozero/exception.hpp @@ -0,0 +1,68 @@ +#ifndef PROTOZERO_EXCEPTION_HPP +#define PROTOZERO_EXCEPTION_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file exception.hpp + * + * @brief Contains the exceptions used in the protozero library. + */ + +#include + +/** + * @brief All parts of the protozero header-only library are in this namespace. + */ +namespace protozero { + +/** + * All exceptions explicitly thrown by the functions of the protozero library + * derive from this exception. + */ +struct exception : std::exception { + /// Returns the explanatory string. + const char *what() const noexcept override { return "pbf exception"; } +}; + +/** + * This exception is thrown when parsing a varint thats larger than allowed. + * This should never happen unless the data is corrupted. + */ +struct varint_too_long_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept override { return "varint too long exception"; } +}; + +/** + * This exception is thrown when the wire type of a pdf field is unknown. + * This should never happen unless the data is corrupted. + */ +struct unknown_pbf_wire_type_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept override { return "unknown pbf field type exception"; } +}; + +/** + * This exception is thrown when we are trying to read a field and there + * are not enough bytes left in the buffer to read it. Almost all functions + * of the pbf_reader class can throw this exception. + * + * This should never happen unless the data is corrupted or you have + * initialized the pbf_reader object with incomplete data. + */ +struct end_of_buffer_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept override { return "end of buffer exception"; } +}; + +} // end namespace protozero + +#endif // PROTOZERO_EXCEPTION_HPP diff --git a/protozero/pbf_builder.hpp b/protozero/pbf_builder.hpp new file mode 100644 index 00000000..548f4cec --- /dev/null +++ b/protozero/pbf_builder.hpp @@ -0,0 +1,139 @@ +#ifndef PROTOZERO_PBF_BUILDER_HPP +#define PROTOZERO_PBF_BUILDER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_builder.hpp + * + * @brief Contains the pbf_builder template class. + */ + +#include + +#include +#include + +namespace protozero { + +/** + * The pbf_builder is used to write PBF formatted messages into a buffer. It + * is based on the pbf_writer class and has all the same methods. The + * difference is that while the pbf_writer class takes an integer tag, + * this template class takes a tag of the template type T. The idea is that + * T will be an enumeration value and this helps reduce the possibility of + * programming errors. + * + * Almost all methods in this class can throw an std::bad_alloc exception if + * the std::string used as a buffer wants to resize. + * + * Read the tutorial to understand how this class is used. + */ +template +class pbf_builder : public pbf_writer { + + static_assert(std::is_same::type>::value, + "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + using enum_type = T; + + pbf_builder(std::string& data) noexcept : + pbf_writer(data) { + } + + template + pbf_builder(pbf_writer& parent_writer, P tag) noexcept : + pbf_writer(parent_writer, pbf_tag_type(tag)) { + } + +/// @cond INTERNAL +#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \ + inline void add_##name(T tag, type value) { \ + pbf_writer::add_##name(pbf_tag_type(tag), value); \ + } + + PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double) + +#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR +/// @endcond + + inline void add_bytes(T tag, const char* value, std::size_t size) { + pbf_writer::add_bytes(pbf_tag_type(tag), value, size); + } + + inline void add_bytes(T tag, const std::string& value) { + pbf_writer::add_bytes(pbf_tag_type(tag), value); + } + + inline void add_string(T tag, const char* value, std::size_t size) { + pbf_writer::add_string(pbf_tag_type(tag), value, size); + } + + inline void add_string(T tag, const std::string& value) { + pbf_writer::add_string(pbf_tag_type(tag), value); + } + + inline void add_string(T tag, const char* value) { + pbf_writer::add_string(pbf_tag_type(tag), value); + } + + inline void add_message(T tag, const char* value, std::size_t size) { + pbf_writer::add_message(pbf_tag_type(tag), value, size); + } + + inline void add_message(T tag, const std::string& value) { + pbf_writer::add_message(pbf_tag_type(tag), value); + } + +/// @cond INTERNAL +#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \ + template \ + inline void add_packed_##name(T tag, InputIterator first, InputIterator last) { \ + pbf_writer::add_packed_##name(pbf_tag_type(tag), first, last); \ + } + + PROTOZERO_WRITER_WRAP_ADD_PACKED(bool) + PROTOZERO_WRITER_WRAP_ADD_PACKED(enum) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(float) + PROTOZERO_WRITER_WRAP_ADD_PACKED(double) + +#undef PROTOZERO_WRITER_WRAP_ADD_PACKED +/// @endcond + +}; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_BUILDER_HPP diff --git a/protozero/pbf_message.hpp b/protozero/pbf_message.hpp new file mode 100644 index 00000000..45f01c14 --- /dev/null +++ b/protozero/pbf_message.hpp @@ -0,0 +1,94 @@ +#ifndef PROTOZERO_PBF_MESSAGE_HPP +#define PROTOZERO_PBF_MESSAGE_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_message.hpp + * + * @brief Contains the pbf_message class. + */ + +#include + +#include +#include + +namespace protozero { + +/** + * This class represents a protobuf message. Either a top-level message or + * a nested sub-message. Top-level messages can be created from any buffer + * with a pointer and length: + * + * @code + * enum class Message : protozero::pbf_tag_type { + * ... + * }; + * + * std::string buffer; + * // fill buffer... + * pbf_message message(buffer.data(), buffer.size()); + * @endcode + * + * Sub-messages are created using get_message(): + * + * @code + * enum class SubMessage : protozero::pbf_tag_type { + * ... + * }; + * + * pbf_message message(...); + * message.next(); + * pbf_message submessage = message.get_message(); + * @endcode + * + * All methods of the pbf_message class except get_bytes() and get_string() + * provide the strong exception guarantee, ie they either succeed or do not + * change the pbf_message object they are called on. Use the get_data() method + * instead of get_bytes() or get_string(), if you need this guarantee. + * + * This template class is based on the pbf_reader class and has all the same + * methods. The difference is that whereever the pbf_reader class takes an + * integer tag, this template class takes a tag of the template type T. + * + * Read the tutorial to understand how this class is used. + */ +template +class pbf_message : public pbf_reader { + + static_assert(std::is_same::type>::value, "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + using enum_type = T; + + template + pbf_message(Args&&... args) noexcept : + pbf_reader(std::forward(args)...) { + } + + inline bool next() { + return pbf_reader::next(); + } + + inline bool next(T tag) { + return pbf_reader::next(pbf_tag_type(tag)); + } + + inline T tag() const noexcept { + return T(pbf_reader::tag()); + } + +}; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_MESSAGE_HPP diff --git a/protozero/pbf_reader.hpp b/protozero/pbf_reader.hpp new file mode 100644 index 00000000..58b38845 --- /dev/null +++ b/protozero/pbf_reader.hpp @@ -0,0 +1,1077 @@ +#ifndef PROTOZERO_PBF_READER_HPP +#define PROTOZERO_PBF_READER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_reader.hpp + * + * @brief Contains the pbf_reader class. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN +# include +#endif + +namespace protozero { + +/** + * This class represents a protobuf message. Either a top-level message or + * a nested sub-message. Top-level messages can be created from any buffer + * with a pointer and length: + * + * @code + * std::string buffer; + * // fill buffer... + * pbf_reader message(buffer.data(), buffer.size()); + * @endcode + * + * Sub-messages are created using get_message(): + * + * @code + * pbf_reader message(...); + * message.next(); + * pbf_reader submessage = message.get_message(); + * @endcode + * + * All methods of the pbf_reader class except get_bytes() and get_string() + * provide the strong exception guarantee, ie they either succeed or do not + * change the pbf_reader object they are called on. Use the get_data() method + * instead of get_bytes() or get_string(), if you need this guarantee. + */ +class pbf_reader { + + // A pointer to the next unread data. + const char *m_data = nullptr; + + // A pointer to one past the end of data. + const char *m_end = nullptr; + + // The wire type of the current field. + pbf_wire_type m_wire_type = pbf_wire_type::unknown; + + // The tag of the current field. + pbf_tag_type m_tag = 0; + + // Copy N bytes from src to dest on little endian machines, on big endian + // swap the bytes in the process. + template + static void copy_or_byteswap(const char* src, void* dest) noexcept { +#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN + memcpy(dest, src, N); +#else + byteswap(src, reinterpret_cast(dest)); +#endif + } + + template + inline T get_fixed() { + T result; + skip_bytes(sizeof(T)); + copy_or_byteswap(m_data - sizeof(T), &result); + return result; + } + +#ifdef PROTOZERO_USE_BARE_POINTER_FOR_PACKED_FIXED + + template + using const_fixed_iterator = const T*; + + template + inline std::pair, const_fixed_iterator> create_fixed_iterator_pair(const char* first, const char* last) { + return std::make_pair(reinterpret_cast(first), + reinterpret_cast(last)); + } + +#else + + template + class const_fixed_iterator : public std::iterator { + + const char* m_data; + const char* m_end; + + public: + + const_fixed_iterator() noexcept : + m_data(nullptr), + m_end(nullptr) { + } + + const_fixed_iterator(const char *data, const char* end) noexcept : + m_data(data), + m_end(end) { + } + + const_fixed_iterator(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator(const_fixed_iterator&&) noexcept = default; + + const_fixed_iterator& operator=(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator& operator=(const_fixed_iterator&&) noexcept = default; + + ~const_fixed_iterator() noexcept = default; + + T operator*() { + T result; + copy_or_byteswap(m_data , &result); + return result; + } + + const_fixed_iterator& operator++() { + m_data += sizeof(T); + return *this; + } + + const_fixed_iterator operator++(int) { + const const_fixed_iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const const_fixed_iterator& rhs) const noexcept { + return m_data == rhs.m_data && m_end == rhs.m_end; + } + + bool operator!=(const const_fixed_iterator& rhs) const noexcept { + return !(*this == rhs); + } + + }; // class const_fixed_iterator + + template + inline std::pair, const_fixed_iterator> create_fixed_iterator_pair(const char* first, const char* last) { + return std::make_pair(const_fixed_iterator(first, last), + const_fixed_iterator(last, last)); + } + +#endif + + template + inline std::pair, const_fixed_iterator> packed_fixed() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + protozero_assert(len % sizeof(T) == 0); + return create_fixed_iterator_pair(m_data-len, m_data); + } + + template inline T get_varint(); + template inline T get_svarint(); + + inline pbf_length_type get_length() { return get_varint(); } + + inline void skip_bytes(pbf_length_type len); + + inline pbf_length_type get_len_and_skip(); + +public: + + /** + * Construct a pbf_reader message from a data pointer and a length. The pointer + * will be stored inside the pbf_reader object, no data is copied. So you must + * make sure the buffer stays valid as long as the pbf_reader object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(const char *data, std::size_t length) noexcept; + + /** + * Construct a pbf_reader message from a data pointer and a length. The pointer + * will be stored inside the pbf_reader object, no data is copied. So you must + * make sure the buffer stays valid as long as the pbf_reader object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(std::pair data) noexcept; + + /** + * Construct a pbf_reader message from a std::string. A pointer to the string + * internals will be stored inside the pbf_reader object, no data is copied. + * So you must make sure the string is unchanged as long as the pbf_reader + * object is used. + * + * The string must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(const std::string& data) noexcept; + + /** + * pbf_reader can be default constructed and behaves like it has an empty + * buffer. + */ + inline pbf_reader() noexcept = default; + + /// pbf_reader messages can be copied trivially. + inline pbf_reader(const pbf_reader&) noexcept = default; + + /// pbf_reader messages can be moved trivially. + inline pbf_reader(pbf_reader&&) noexcept = default; + + /// pbf_reader messages can be copied trivially. + inline pbf_reader& operator=(const pbf_reader& other) noexcept = default; + + /// pbf_reader messages can be moved trivially. + inline pbf_reader& operator=(pbf_reader&& other) noexcept = default; + + inline ~pbf_reader() = default; + + /** + * In a boolean context the pbf_reader class evaluates to `true` if there are + * still fields available and to `false` if the last field has been read. + */ + inline operator bool() const noexcept; + + /** + * Return the length in bytes of the current message. If you have + * already called next() and/or any of the get_*() functions, this will + * return the remaining length. + * + * This can, for instance, be used to estimate the space needed for a + * buffer. Of course you have to know reasonably well what data to expect + * and how it is encoded for this number to have any meaning. + */ + std::size_t length() const noexcept { + return std::size_t(m_end - m_data); + } + + /** + * Set next field in the message as the current field. This is usually + * called in a while loop: + * + * @code + * pbf_reader message(...); + * while (message.next()) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field, `false` if not. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now. + */ + inline bool next(); + + /** + * Set next field with given tag in the message as the current field. + * Fields with other tags are skipped. This is usually called in a while + * loop for repeated fields: + * + * @code + * pbf_reader message(...); + * while (message.next(17)) { + * // handle field + * } + * @endcode + * + * or you can call it just once to get the one field with this tag: + * + * @code + * pbf_reader message(...); + * if (message.next(17)) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field with this tag. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now with the given tag. + */ + inline bool next(pbf_tag_type tag); + + /** + * The tag of the current field. The tag is the field number from the + * description in the .proto file. + * + * Call next() before calling this function to set the current field. + * + * @returns tag of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline pbf_tag_type tag() const noexcept; + + /** + * Get the wire type of the current field. The wire types are: + * + * * 0 - varint + * * 1 - 64 bit + * * 2 - length-delimited + * * 5 - 32 bit + * + * All other types are illegal. + * + * Call next() before calling this function to set the current field. + * + * @returns wire type of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline pbf_wire_type wire_type() const noexcept; + + /** + * Check the wire type of the current field. + * + * @returns `true` if the current field has the given wire type. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline bool has_wire_type(pbf_wire_type type) const noexcept; + + /** + * Consume the current field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @post The current field was consumed and there is no current field now. + */ + inline void skip(); + + ///@{ + /** + * @name Scalar field accessor functions + */ + + /** + * Consume and return value of current "bool" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bool". + * @post The current field was consumed and there is no current field now. + */ + inline bool get_bool(); + + /** + * Consume and return value of current "enum" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "enum". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_enum() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "int32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_int32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "sint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_sint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint(); + } + + /** + * Consume and return value of current "uint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint32". + * @post The current field was consumed and there is no current field now. + */ + inline uint32_t get_uint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "int64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_int64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "sint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_sint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint(); + } + + /** + * Consume and return value of current "uint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint64". + * @post The current field was consumed and there is no current field now. + */ + inline uint64_t get_uint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "fixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed32". + * @post The current field was consumed and there is no current field now. + */ + inline uint32_t get_fixed32(); + + /** + * Consume and return value of current "sfixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_sfixed32(); + + /** + * Consume and return value of current "fixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed64". + * @post The current field was consumed and there is no current field now. + */ + inline uint64_t get_fixed64(); + + /** + * Consume and return value of current "sfixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_sfixed64(); + + /** + * Consume and return value of current "float" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "float". + * @post The current field was consumed and there is no current field now. + */ + inline float get_float(); + + /** + * Consume and return value of current "double" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "double". + * @post The current field was consumed and there is no current field now. + */ + inline double get_double(); + + /** + * Consume and return value of current "bytes" or "string" field. + * + * @returns A pair with a pointer to the data and the length of the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes" or "string". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_data(); + + /** + * Consume and return value of current "bytes" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes". + * @post The current field was consumed and there is no current field now. + */ + inline std::string get_bytes(); + + /** + * Consume and return value of current "string" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "string". + * @post The current field was consumed and there is no current field now. + */ + inline std::string get_string(); + + /** + * Consume and return value of current "message" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "message". + * @post The current field was consumed and there is no current field now. + */ + inline pbf_reader get_message() { + return pbf_reader(get_data()); + } + + ///@} + +private: + + template + class const_varint_iterator : public std::iterator { + + protected: + + const char* m_data; + const char* m_end; + + public: + + const_varint_iterator() noexcept : + m_data(nullptr), + m_end(nullptr) { + } + + const_varint_iterator(const char *data, const char* end) noexcept : + m_data(data), + m_end(end) { + } + + const_varint_iterator(const const_varint_iterator&) noexcept = default; + const_varint_iterator(const_varint_iterator&&) noexcept = default; + + const_varint_iterator& operator=(const const_varint_iterator&) noexcept = default; + const_varint_iterator& operator=(const_varint_iterator&&) noexcept = default; + + ~const_varint_iterator() noexcept = default; + + T operator*() { + const char* d = m_data; // will be thrown away + return static_cast(decode_varint(&d, m_end)); + } + + const_varint_iterator& operator++() { + // Ignore the result, we call decode_varint() just for the + // side-effect of updating m_data. + decode_varint(&m_data, m_end); + return *this; + } + + const_varint_iterator operator++(int) { + const const_varint_iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const const_varint_iterator& rhs) const noexcept { + return m_data == rhs.m_data && m_end == rhs.m_end; + } + + bool operator!=(const const_varint_iterator& rhs) const noexcept { + return !(*this == rhs); + } + + }; // class const_varint_iterator + + template + class const_svarint_iterator : public const_varint_iterator { + + public: + + const_svarint_iterator() noexcept : + const_varint_iterator() { + } + + const_svarint_iterator(const char *data, const char* end) noexcept : + const_varint_iterator(data, end) { + } + + const_svarint_iterator(const const_svarint_iterator&) = default; + const_svarint_iterator(const_svarint_iterator&&) = default; + + const_svarint_iterator& operator=(const const_svarint_iterator&) = default; + const_svarint_iterator& operator=(const_svarint_iterator&&) = default; + + ~const_svarint_iterator() = default; + + T operator*() { + const char* d = this->m_data; // will be thrown away + return static_cast(decode_zigzag64(decode_varint(&d, this->m_end))); + } + + const_svarint_iterator& operator++() { + // Ignore the result, we call decode_varint() just for the + // side-effect of updating m_data. + decode_varint(&this->m_data, this->m_end); + return *this; + } + + const_svarint_iterator operator++(int) { + const const_svarint_iterator tmp(*this); + ++(*this); + return tmp; + } + + }; // class const_svarint_iterator + +public: + + /// Forward iterator for iterating over bool (int32 varint) values. + typedef const_varint_iterator< int32_t> const_bool_iterator; + + /// Forward iterator for iterating over enum (int32 varint) values. + typedef const_varint_iterator< int32_t> const_enum_iterator; + + /// Forward iterator for iterating over int32 (varint) values. + typedef const_varint_iterator< int32_t> const_int32_iterator; + + /// Forward iterator for iterating over sint32 (varint) values. + typedef const_svarint_iterator const_sint32_iterator; + + /// Forward iterator for iterating over uint32 (varint) values. + typedef const_varint_iterator const_uint32_iterator; + + /// Forward iterator for iterating over int64 (varint) values. + typedef const_varint_iterator< int64_t> const_int64_iterator; + + /// Forward iterator for iterating over sint64 (varint) values. + typedef const_svarint_iterator const_sint64_iterator; + + /// Forward iterator for iterating over uint64 (varint) values. + typedef const_varint_iterator const_uint64_iterator; + + ///@{ + /** + * @name Repeated packed field accessor functions + */ + + /** + * Consume current "repeated packed bool" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed bool". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_bool(); + + /** + * Consume current "repeated packed enum" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed enum". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_enum(); + + /** + * Consume current "repeated packed int32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_int32(); + + /** + * Consume current "repeated packed sint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_sint32(); + + /** + * Consume current "repeated packed uint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_uint32(); + + /** + * Consume current "repeated packed int64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_int64(); + + /** + * Consume current "repeated packed sint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_sint64(); + + /** + * Consume current "repeated packed uint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_uint64(); + + /** + * Consume current "repeated packed fixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed32". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_fixed32() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed sfixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed32". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_sfixed32() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed fixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed64". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_fixed64() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed sfixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed64". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_sfixed64() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed float" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed float". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_float() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed double" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed double". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_double() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + ///@} + +}; // class pbf_reader + +pbf_reader::pbf_reader(const char *data, std::size_t length) noexcept + : m_data(data), + m_end(data + length), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::pbf_reader(std::pair data) noexcept + : m_data(data.first), + m_end(data.first + data.second), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::pbf_reader(const std::string& data) noexcept + : m_data(data.data()), + m_end(data.data() + data.size()), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::operator bool() const noexcept { + return m_data < m_end; +} + +bool pbf_reader::next() { + if (m_data == m_end) { + return false; + } + + auto value = get_varint(); + m_tag = value >> 3; + + // tags 0 and 19000 to 19999 are not allowed as per + // https://developers.google.com/protocol-buffers/docs/proto + protozero_assert(((m_tag > 0 && m_tag < 19000) || (m_tag > 19999 && m_tag <= ((1 << 29) - 1))) && "tag out of range"); + + m_wire_type = pbf_wire_type(value & 0x07); + switch (m_wire_type) { + case pbf_wire_type::varint: + case pbf_wire_type::fixed64: + case pbf_wire_type::length_delimited: + case pbf_wire_type::fixed32: + break; + default: + throw unknown_pbf_wire_type_exception(); + } + + return true; +} + +bool pbf_reader::next(pbf_tag_type requested_tag) { + while (next()) { + if (m_tag == requested_tag) { + return true; + } else { + skip(); + } + } + return false; +} + +pbf_tag_type pbf_reader::tag() const noexcept { + return m_tag; +} + +pbf_wire_type pbf_reader::wire_type() const noexcept { + return m_wire_type; +} + +bool pbf_reader::has_wire_type(pbf_wire_type type) const noexcept { + return wire_type() == type; +} + +void pbf_reader::skip_bytes(pbf_length_type len) { + if (m_data + len > m_end) { + throw end_of_buffer_exception(); + } + m_data += len; + +// In debug builds reset the tag to zero so that we can detect (some) +// wrong code. +#ifndef NDEBUG + m_tag = 0; +#endif +} + +void pbf_reader::skip() { + protozero_assert(tag() != 0 && "call next() before calling skip()"); + switch (wire_type()) { + case pbf_wire_type::varint: + (void)get_uint32(); // called for the side-effect of skipping value + break; + case pbf_wire_type::fixed64: + skip_bytes(8); + break; + case pbf_wire_type::length_delimited: + skip_bytes(get_length()); + break; + case pbf_wire_type::fixed32: + skip_bytes(4); + break; + default: + protozero_assert(false && "can not be here because next() should have thrown already"); + } +} + +pbf_length_type pbf_reader::get_len_and_skip() { + auto len = get_length(); + skip_bytes(len); + return len; +} + +template +T pbf_reader::get_varint() { + return static_cast(decode_varint(&m_data, m_end)); +} + +template +T pbf_reader::get_svarint() { + protozero_assert((has_wire_type(pbf_wire_type::varint) || has_wire_type(pbf_wire_type::length_delimited)) && "not a varint"); + return static_cast(decode_zigzag64(decode_varint(&m_data, m_end))); +} + +uint32_t pbf_reader::get_fixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +int32_t pbf_reader::get_sfixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +uint64_t pbf_reader::get_fixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +int64_t pbf_reader::get_sfixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +float pbf_reader::get_float() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +double pbf_reader::get_double() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +bool pbf_reader::get_bool() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + protozero_assert((*m_data & 0x80) == 0 && "not a 1 byte varint"); + skip_bytes(1); + return m_data[-1] != 0; // -1 okay because we incremented m_data the line before +} + +std::pair pbf_reader::get_data() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message"); + auto len = get_len_and_skip(); + return std::make_pair(m_data-len, len); +} + +std::string pbf_reader::get_bytes() { + auto d = get_data(); + return std::string(d.first, d.second); +} + +std::string pbf_reader::get_string() { + return get_bytes(); +} + +std::pair pbf_reader::get_packed_bool() { + return get_packed_int32(); +} + +std::pair pbf_reader::get_packed_enum() { + return get_packed_int32(); +} + +std::pair pbf_reader::get_packed_int32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_int32_iterator(m_data-len, m_data), + pbf_reader::const_int32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_uint32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_uint32_iterator(m_data-len, m_data), + pbf_reader::const_uint32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_sint32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_sint32_iterator(m_data-len, m_data), + pbf_reader::const_sint32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_int64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_int64_iterator(m_data-len, m_data), + pbf_reader::const_int64_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_uint64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_uint64_iterator(m_data-len, m_data), + pbf_reader::const_uint64_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_sint64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_sint64_iterator(m_data-len, m_data), + pbf_reader::const_sint64_iterator(m_data, m_data)); +} + +} // end namespace protozero + +#endif // PROTOZERO_PBF_READER_HPP diff --git a/protozero/pbf_writer.hpp b/protozero/pbf_writer.hpp new file mode 100644 index 00000000..422e1477 --- /dev/null +++ b/protozero/pbf_writer.hpp @@ -0,0 +1,837 @@ +#ifndef PROTOZERO_PBF_WRITER_HPP +#define PROTOZERO_PBF_WRITER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_writer.hpp + * + * @brief Contains the pbf_writer class. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN +# include +#endif + +namespace protozero { + +namespace detail { + + template class packed_field_varint; + template class packed_field_svarint; + template class packed_field_fixed; + +} // end namespace detail + +/** + * The pbf_writer is used to write PBF formatted messages into a buffer. + * + * Almost all methods in this class can throw an std::bad_alloc exception if + * the std::string used as a buffer wants to resize. + */ +class pbf_writer { + + // A pointer to a string buffer holding the data already written to the + // PBF message. For default constructed writers or writers that have been + // rolled back, this is a nullptr. + std::string* m_data; + + // A pointer to a parent writer object if this is a submessage. If this + // is a top-level writer, it is a nullptr. + pbf_writer* m_parent_writer; + + // This is usually 0. If there is an open submessage, this is set in the + // parent to the rollback position, ie. the last position before the + // submessage was started. This is the position where the header of the + // submessage starts. + std::size_t m_rollback_pos = 0; + + // This is usually 0. If there is an open submessage, this is set in the + // parent to the position where the data of the submessage is written to. + std::size_t m_pos = 0; + + inline void add_varint(uint64_t value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + write_varint(std::back_inserter(*m_data), value); + } + + inline void add_field(pbf_tag_type tag, pbf_wire_type type) { + protozero_assert(((tag > 0 && tag < 19000) || (tag > 19999 && tag <= ((1 << 29) - 1))) && "tag out of range"); + uint32_t b = (tag << 3) | uint32_t(type); + add_varint(b); + } + + inline void add_tagged_varint(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::varint); + add_varint(value); + } + + template + inline void add_fixed(T value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); +#if PROTOZERO_BYTE_ORDER == PROTOZERO_LITTLE_ENDIAN + m_data->append(reinterpret_cast(&value), sizeof(T)); +#else + auto size = m_data->size(); + m_data->resize(size + sizeof(T)); + byteswap(reinterpret_cast(&value), const_cast(m_data->data() + size)); +#endif + } + + template + inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::input_iterator_tag) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_fixed(*first++); + } + } + + template + inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::forward_iterator_tag) { + if (first == last) { + return; + } + + auto length = std::distance(first, last); + add_length_varint(tag, sizeof(T) * pbf_length_type(length)); + reserve(sizeof(T) * std::size_t(length)); + + while (first != last) { + add_fixed(*first++); + } + } + + template + inline void add_packed_varint(pbf_tag_type tag, It first, It last) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_varint(uint64_t(*first++)); + } + } + + template + inline void add_packed_svarint(pbf_tag_type tag, It first, It last) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_varint(encode_zigzag64(*first++)); + } + } + + // The number of bytes to reserve for the varint holding the length of + // a length-delimited field. The length has to fit into pbf_length_type, + // and a varint needs 8 bit for every 7 bit. + static const int reserve_bytes = sizeof(pbf_length_type) * 8 / 7 + 1; + + // If m_rollpack_pos is set to this special value, it means that when + // the submessage is closed, nothing needs to be done, because the length + // of the submessage has already been written correctly. + static const std::size_t size_is_known = std::numeric_limits::max(); + + inline void open_submessage(pbf_tag_type tag, std::size_t size) { + protozero_assert(m_pos == 0); + protozero_assert(m_data); + if (size == 0) { + m_rollback_pos = m_data->size(); + add_field(tag, pbf_wire_type::length_delimited); + m_data->append(std::size_t(reserve_bytes), '\0'); + } else { + m_rollback_pos = size_is_known; + add_length_varint(tag, pbf_length_type(size)); + reserve(size); + } + m_pos = m_data->size(); + } + + inline void rollback_submessage() { + protozero_assert(m_pos != 0); + protozero_assert(m_rollback_pos != size_is_known); + protozero_assert(m_data); + m_data->resize(m_rollback_pos); + m_pos = 0; + } + + inline void commit_submessage() { + protozero_assert(m_pos != 0); + protozero_assert(m_rollback_pos != size_is_known); + protozero_assert(m_data); + auto length = pbf_length_type(m_data->size() - m_pos); + + protozero_assert(m_data->size() >= m_pos - reserve_bytes); + auto n = write_varint(m_data->begin() + long(m_pos) - reserve_bytes, length); + + m_data->erase(m_data->begin() + long(m_pos) - reserve_bytes + n, m_data->begin() + long(m_pos)); + m_pos = 0; + } + + inline void close_submessage() { + protozero_assert(m_data); + if (m_pos == 0 || m_rollback_pos == size_is_known) { + return; + } + if (m_data->size() - m_pos == 0) { + rollback_submessage(); + } else { + commit_submessage(); + } + } + + inline void add_length_varint(pbf_tag_type tag, pbf_length_type length) { + add_field(tag, pbf_wire_type::length_delimited); + add_varint(length); + } + +public: + + /** + * Create a writer using the given string as a data store. The pbf_writer + * stores a reference to that string and adds all data to it. The string + * doesn't have to be empty. The pbf_writer will just append data. + */ + inline explicit pbf_writer(std::string& data) noexcept : + m_data(&data), + m_parent_writer(nullptr), + m_pos(0) { + } + + /** + * Create a writer without a data store. In this form the writer can not + * be used! + */ + inline pbf_writer() noexcept : + m_data(nullptr), + m_parent_writer(nullptr), + m_pos(0) { + } + + /** + * Construct a pbf_writer for a submessage from the pbf_writer of the + * parent message. + * + * @param parent_writer The pbf_writer + * @param tag Tag (field number) of the field that will be written + * @param size Optional size of the submessage in bytes (use 0 for unknown). + * Setting this allows some optimizations but is only possible in + * a few very specific cases. + */ + inline pbf_writer(pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size=0) : + m_data(parent_writer.m_data), + m_parent_writer(&parent_writer), + m_pos(0) { + m_parent_writer->open_submessage(tag, size); + } + + /// A pbf_writer object can be copied + pbf_writer(const pbf_writer&) noexcept = default; + + /// A pbf_writer object can be copied + pbf_writer& operator=(const pbf_writer&) noexcept = default; + + /// A pbf_writer object can be moved + inline pbf_writer(pbf_writer&&) noexcept = default; + + /// A pbf_writer object can be moved + inline pbf_writer& operator=(pbf_writer&&) noexcept = default; + + inline ~pbf_writer() { + if (m_parent_writer) { + m_parent_writer->close_submessage(); + } + } + + /** + * Reserve size bytes in the underlying message store in addition to + * whatever the message store already holds. So unlike + * the `std::string::reserve()` method this is not an absolute size, + * but additional memory that should be reserved. + * + * @param size Number of bytes to reserve in underlying message store. + */ + void reserve(std::size_t size) { + protozero_assert(m_data); + m_data->reserve(m_data->size() + size); + } + + inline void rollback() { + protozero_assert(m_parent_writer && "you can't call rollback() on a pbf_writer without a parent"); + protozero_assert(m_pos == 0 && "you can't call rollback() on a pbf_writer that has an open nested submessage"); + m_parent_writer->rollback_submessage(); + m_data = nullptr; + } + + ///@{ + /** + * @name Scalar field writer functions + */ + + /** + * Add "bool" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_bool(pbf_tag_type tag, bool value) { + add_field(tag, pbf_wire_type::varint); + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + m_data->append(1, value); + } + + /** + * Add "enum" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_enum(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "int32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_int32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "sint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sint32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, encode_zigzag32(value)); + } + + /** + * Add "uint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_uint32(pbf_tag_type tag, uint32_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "int64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_int64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "sint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sint64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, encode_zigzag64(value)); + } + + /** + * Add "uint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_uint64(pbf_tag_type tag, uint64_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "fixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_fixed32(pbf_tag_type tag, uint32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "sfixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sfixed32(pbf_tag_type tag, int32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "fixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_fixed64(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "sfixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sfixed64(pbf_tag_type tag, int64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "float" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_float(pbf_tag_type tag, float value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "double" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_double(pbf_tag_type tag, double value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + inline void add_bytes(pbf_tag_type tag, const char* value, std::size_t size) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + protozero_assert(size <= std::numeric_limits::max()); + add_length_varint(tag, pbf_length_type(size)); + m_data->append(value, size); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_bytes(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + inline void add_string(pbf_tag_type tag, const char* value, std::size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_string(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. Bytes from the value are written until + * a null byte is encountered. The null byte is not added. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + */ + inline void add_string(pbf_tag_type tag, const char* value) { + add_bytes(tag, value, std::strlen(value)); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to message to be written + * @param size Length of the message + */ + inline void add_message(pbf_tag_type tag, const char* value, std::size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written. The value must be a complete message. + */ + inline void add_message(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + ///@} + + ///@{ + /** + * @name Repeated packed field writer functions + */ + + /** + * Add "repeated packed bool" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to bool. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_bool(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed enum" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_enum(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_int32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sint32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_uint32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_int64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sint64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_uint64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed fixed32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_fixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed sfixed32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sfixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed fixed64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_fixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed sfixed64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sfixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed float" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to float. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_float(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed double" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to double. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_double(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + ///@} + + template friend class detail::packed_field_varint; + template friend class detail::packed_field_svarint; + template friend class detail::packed_field_fixed; + +}; // class pbf_writer + +namespace detail { + + class packed_field { + + protected: + + pbf_writer m_writer; + + public: + + packed_field(pbf_writer& parent_writer, pbf_tag_type tag) : + m_writer(parent_writer, tag) { + } + + packed_field(pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size) : + m_writer(parent_writer, tag, size) { + } + + void rollback() { + m_writer.rollback(); + } + + }; // class packed_field + + template + class packed_field_fixed : public packed_field { + + public: + + packed_field_fixed(pbf_writer& parent_writer, pbf_tag_type tag) : + packed_field(parent_writer, tag) { + } + + packed_field_fixed(pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size) : + packed_field(parent_writer, tag, size * sizeof(T)) { + } + + void add_element(T value) { + m_writer.add_fixed(value); + } + + }; // class packed_field_fixed + + template + class packed_field_varint : public packed_field { + + public: + + packed_field_varint(pbf_writer& parent_writer, pbf_tag_type tag) : + packed_field(parent_writer, tag) { + } + + void add_element(T value) { + m_writer.add_varint(uint64_t(value)); + } + + }; // class packed_field_varint + + template + class packed_field_svarint : public packed_field { + + public: + + packed_field_svarint(pbf_writer& parent_writer, pbf_tag_type tag) : + packed_field(parent_writer, tag) { + } + + void add_element(T value) { + m_writer.add_varint(encode_zigzag64(value)); + } + + }; // class packed_field_svarint + +} // end namespace detail + +using packed_field_bool = detail::packed_field_varint; +using packed_field_enum = detail::packed_field_varint; +using packed_field_int32 = detail::packed_field_varint; +using packed_field_sint32 = detail::packed_field_svarint; +using packed_field_uint32 = detail::packed_field_varint; +using packed_field_int64 = detail::packed_field_varint; +using packed_field_sint64 = detail::packed_field_svarint; +using packed_field_uint64 = detail::packed_field_varint; +using packed_field_fixed32 = detail::packed_field_fixed; +using packed_field_sfixed32 = detail::packed_field_fixed; +using packed_field_fixed64 = detail::packed_field_fixed; +using packed_field_sfixed64 = detail::packed_field_fixed; +using packed_field_float = detail::packed_field_fixed; +using packed_field_double = detail::packed_field_fixed; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_WRITER_HPP diff --git a/protozero/types.hpp b/protozero/types.hpp new file mode 100644 index 00000000..6856b3dd --- /dev/null +++ b/protozero/types.hpp @@ -0,0 +1,49 @@ +#ifndef PROTOZERO_TYPES_HPP +#define PROTOZERO_TYPES_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file types.hpp + * + * @brief Contains the declaration of low-level types used in the pbf format. + */ + +#include + +namespace protozero { + + /** + * The type used for field tags (field numbers). + */ + typedef uint32_t pbf_tag_type; + + /** + * The type used to encode type information. + * See the table on + * https://developers.google.com/protocol-buffers/docs/encoding + */ + enum class pbf_wire_type : uint32_t { + varint = 0, // int32/64, uint32/64, sint32/64, bool, enum + fixed64 = 1, // fixed64, sfixed64, double + length_delimited = 2, // string, bytes, embedded messages, + // packed repeated fields + fixed32 = 5, // fixed32, sfixed32, float + unknown = 99 // used for default setting in this library + }; + + /** + * The type used for length values, such as the length of a field. + */ + typedef uint32_t pbf_length_type; + +} // end namespace protozero + +#endif // PROTOZERO_TYPES_HPP diff --git a/protozero/varint.hpp b/protozero/varint.hpp new file mode 100644 index 00000000..4242df95 --- /dev/null +++ b/protozero/varint.hpp @@ -0,0 +1,132 @@ +#ifndef PROTOZERO_VARINT_HPP +#define PROTOZERO_VARINT_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file varint.hpp + * + * @brief Contains low-level varint and zigzag encoding and decoding functions. + */ + +#include + +#include + +namespace protozero { + +/** + * The maximum length of a 64bit varint. + */ +constexpr const int8_t max_varint_length = sizeof(uint64_t) * 8 / 7 + 1; + +// from https://github.com/facebook/folly/blob/master/folly/Varint.h +/** + * Decode a 64bit varint. + * + * Strong exception guarantee: if there is an exception the data pointer will + * not be changed. + * + * @param[in,out] data Pointer to pointer to the input data. After the function + * returns this will point to the next data to be read. + * @param[in] end Pointer one past the end of the input data. + * @returns The decoded integer + * @throws varint_too_long_exception if the varint is longer then the maximum + * length that would fit in a 64bit int. Usually this means your data + * is corrupted or you are trying to read something as a varint that + * isn't. + * @throws end_of_buffer_exception if the *end* of the buffer was reached + * before the end of the varint. + */ +inline uint64_t decode_varint(const char** data, const char* end) { + const int8_t* begin = reinterpret_cast(*data); + const int8_t* iend = reinterpret_cast(end); + const int8_t* p = begin; + uint64_t val = 0; + + if (iend - begin >= max_varint_length) { // fast path + do { + int64_t b; + b = *p++; val = uint64_t((b & 0x7f) ); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 7); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 14); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 21); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 28); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 35); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 42); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 49); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 56); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 63); if (b >= 0) break; + throw varint_too_long_exception(); + } while (false); + } else { + int shift = 0; + while (p != iend && *p < 0) { + val |= uint64_t(*p++ & 0x7f) << shift; + shift += 7; + } + if (p == iend) { + throw end_of_buffer_exception(); + } + val |= uint64_t(*p++) << shift; + } + + *data = reinterpret_cast(p); + return val; +} + +/** + * Varint-encode a 64bit integer. + */ +template +inline int write_varint(OutputIterator data, uint64_t value) { + int n=1; + + while (value >= 0x80) { + *data++ = char((value & 0x7f) | 0x80); + value >>= 7; + ++n; + } + *data++ = char(value); + + return n; +} + +/** + * ZigZag encodes a 32 bit integer. + */ +inline uint32_t encode_zigzag32(int32_t value) noexcept { + return (static_cast(value) << 1) ^ (static_cast(value >> 31)); +} + +/** + * ZigZag encodes a 64 bit integer. + */ +inline uint64_t encode_zigzag64(int64_t value) noexcept { + return (static_cast(value) << 1) ^ (static_cast(value >> 63)); +} + +/** + * Decodes a 32 bit ZigZag-encoded integer. + */ +inline int32_t decode_zigzag32(uint32_t value) noexcept { + return int32_t(value >> 1) ^ -int32_t(value & 1); +} + +/** + * Decodes a 64 bit ZigZag-encoded integer. + */ +inline int64_t decode_zigzag64(uint64_t value) noexcept { + return int64_t(value >> 1) ^ -int64_t(value & 1); +} + +} // end namespace protozero + +#endif // PROTOZERO_VARINT_HPP diff --git a/protozero/version.hpp b/protozero/version.hpp new file mode 100644 index 00000000..7b60e2ed --- /dev/null +++ b/protozero/version.hpp @@ -0,0 +1,22 @@ +#ifndef PROTOZERO_VERSION_HPP +#define PROTOZERO_VERSION_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#define PROTOZERO_VERSION_MAJOR 1 +#define PROTOZERO_VERSION_MINOR 3 +#define PROTOZERO_VERSION_PATCH 0 + +#define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH) + +#define PROTOZERO_VERSION_STRING "1.3.0" + + +#endif // PROTOZERO_VERSION_HPP