diff options
| author | Stefan Boberg <[email protected]> | 2025-10-22 17:57:29 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-10-22 17:57:29 +0200 |
| commit | 5c139e2d8a260544bc5e730de0440edbab4b0f03 (patch) | |
| tree | b477208925fe3b373d4833460b90d61a8051cf05 | |
| parent | 5.7.7-pre3 (diff) | |
| download | zen-5c139e2d8a260544bc5e730de0440edbab4b0f03.tar.xz zen-5c139e2d8a260544bc5e730de0440edbab4b0f03.zip | |
add support for OTLP logging/tracing (#599)
- adds `zentelemetry` project which houses new functionality for serializing logs and traces in OpenTelemetry Protocol format (OTLP)
- moved existing stats functionality from `zencore` to `zentelemetry`
- adds `TRefCounted<T>` for vtable-less refcounting
- adds `MemoryArena` class which allows for linear allocation of memory from chunks
- adds `protozero` which is used to encode OTLP protobuf messages
63 files changed, 8872 insertions, 28 deletions
diff --git a/src/zenbase/include/zenbase/refcount.h b/src/zenbase/include/zenbase/refcount.h index 6ad49cba2..40ad7bca5 100644 --- a/src/zenbase/include/zenbase/refcount.h +++ b/src/zenbase/include/zenbase/refcount.h @@ -10,6 +10,9 @@ namespace zen { /** * Helper base class for reference counted objects using intrusive reference counting + * + * When the reference count reaches zero, the object deletes itself. This class relies + * on having a virtual destructor to ensure proper cleanup of derived classes. */ class RefCounted { @@ -17,7 +20,7 @@ public: RefCounted() = default; virtual ~RefCounted() = default; - inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); } + inline uint32_t AddRef() const noexcept { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); } inline uint32_t Release() const { const uint32_t RefCount = AtomicDecrement(const_cast<RefCounted*>(this)->m_RefCount); @@ -43,6 +46,48 @@ private: }; /** + * Template helper base class for reference counted objects using intrusive reference counting. + * + * NOTE: Unlike RefCounted, this class deletes the derived type when the reference count reaches zero. + * It has no virtual destructor, so it's important that you either don't derive from it further, + * or ensure that the derived class has a virtual destructor. + */ + +template<typename T> +class TRefCounted +{ +public: + TRefCounted() = default; + ~TRefCounted() = default; + + inline uint32_t AddRef() const noexcept { return AtomicIncrement(const_cast<TRefCounted<T>*>(this)->m_RefCount); } + inline uint32_t Release() const + { + const uint32_t RefCount = AtomicDecrement(const_cast<TRefCounted<T>*>(this)->m_RefCount); + if (RefCount == 0) + { + const_cast<T*>(static_cast<const T*>(this))->DeleteThis(); + } + return RefCount; + } + + // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it + + TRefCounted(const TRefCounted&) = delete; + TRefCounted(TRefCounted&&) = delete; + TRefCounted& operator=(const TRefCounted&) = delete; + TRefCounted& operator=(TRefCounted&&) = delete; + +protected: + inline uint32_t RefCount() const { return m_RefCount; } + + void DeleteThis() const noexcept { delete static_cast<const T*>(this); } + +private: + uint32_t m_RefCount = 0; +}; + +/** * Smart pointer for classes derived from RefCounted */ @@ -127,7 +172,8 @@ class Ref { public: inline Ref() = default; - inline Ref(const Ref& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } + inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } + inline Ref(const Ref& Rhs) noexcept : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); } inline explicit Ref(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); } inline ~Ref() { m_Ref && m_Ref->Release(); } @@ -170,7 +216,6 @@ public: } return *this; } - inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; } private: T* m_Ref = nullptr; diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h index 63779407e..1b2d382ee 100644 --- a/src/zencore/include/zencore/iobuffer.h +++ b/src/zencore/include/zencore/iobuffer.h @@ -32,6 +32,7 @@ enum class ZenContentType : uint8_t kPNG = 12, kIcon = 13, kXML = 14, + kProtobuf = 15, kCOUNT }; @@ -73,6 +74,8 @@ ToString(ZenContentType ContentType) return "icon"sv; case ZenContentType::kXML: return "xml"sv; + case ZenContentType::kProtobuf: + return "protobuf"sv; } } diff --git a/src/zencore/include/zencore/memory/memoryarena.h b/src/zencore/include/zencore/memory/memoryarena.h new file mode 100644 index 000000000..551415aac --- /dev/null +++ b/src/zencore/include/zencore/memory/memoryarena.h @@ -0,0 +1,104 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/thread.h> +#include <vector> + +namespace zen { + +/** + * Chunked linear memory allocator + * + * Supports fast allocation of many small objects with minimal overhead. + * All allocations are freed when the arena is destroyed, therefore there + * is no support for individual deallocation. + * + * For convenience, we include operator new/delete overloads below which + * take a MemoryArena reference as a placement argument. + * + * This allocator is thread-safe. + */ +class MemoryArena +{ +public: + MemoryArena() = default; + ~MemoryArena(); + + void* AllocateAligned(size_t ByteCount, size_t align); + void* AllocateAlignedWithOffset(size_t ByteCount, size_t align, size_t offset); + void* Allocate(size_t Size); + const char* DuplicateString(std::string_view Str); + + MemoryArena(const MemoryArena&) = delete; + MemoryArena& operator=(const MemoryArena&) = delete; + +private: + static constexpr size_t ChunkSize = 16 * 1024; + // TODO: should just chain the memory blocks together and avoid this + // vector altogether, saving us a memory allocation + std::vector<uint8_t*> m_Chunks; + uint8_t* m_CurrentChunk = nullptr; + size_t m_CurrentOffset = 0; + RwLock m_Lock; +}; + +// Allocator suitable for use with EASTL + +struct ArenaAlloc +{ + ArenaAlloc(const char* name_opt = nullptr) = delete; + ArenaAlloc(MemoryArena& Arena) : m_Arena(&Arena) {} + + inline void* allocate(size_t bytes, int flags = 0) + { + ZEN_UNUSED(flags); + return m_Arena->Allocate(bytes); + } + + inline void* allocate(size_t bytes, size_t align, size_t offset, int flags = 0) + { + ZEN_UNUSED(flags); + if (offset == 0) + { + return m_Arena->AllocateAligned(bytes, align); + } + else + { + return m_Arena->AllocateAlignedWithOffset(bytes, align, offset); + } + } + + void deallocate(void* p, size_t n) { ZEN_UNUSED(p, n); } + +private: + MemoryArena* m_Arena = nullptr; +}; + +} // namespace zen + +inline void* +operator new(size_t Size, zen::MemoryArena& Arena) +{ + return Arena.Allocate(Size); +} + +inline void +operator delete(void* Ptr, zen::MemoryArena& Arena) +{ + // Arena will clean up all allocations when it's destroyed + ZEN_UNUSED(Ptr, Arena); +} + +inline void* +operator new[](size_t Size, zen::MemoryArena& Arena) +{ + return Arena.Allocate(Size); +} + +inline void +operator delete[](void* Ptr, zen::MemoryArena& Arena) +{ + // Arena will clean up all allocations when it's destroyed + ZEN_UNUSED(Ptr, Arena); +} diff --git a/src/zencore/memory/memoryarena.cpp b/src/zencore/memory/memoryarena.cpp new file mode 100644 index 000000000..9c907a66d --- /dev/null +++ b/src/zencore/memory/memoryarena.cpp @@ -0,0 +1,126 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/memory/memoryarena.h> + +namespace zen { + +MemoryArena::~MemoryArena() +{ + for (auto Chunk : m_Chunks) + delete[] Chunk; +} + +void* +MemoryArena::AllocateAligned(size_t ByteCount, size_t align) +{ + if (ByteCount == 0) + { + return nullptr; + } + + void* Ptr = nullptr; + + m_Lock.WithExclusiveLock([&] { + size_t AlignedOffset = (m_CurrentOffset + (align - 1)) & ~(align - 1); + + if (m_CurrentChunk == nullptr || AlignedOffset + ByteCount > ChunkSize) + { + uint8_t* NewChunk = new uint8_t[ChunkSize]; + if (!NewChunk) + { + return; + } + + m_Chunks.push_back(NewChunk); + m_CurrentChunk = NewChunk; + AlignedOffset = 0; + } + + Ptr = m_CurrentChunk + AlignedOffset; + m_CurrentOffset = AlignedOffset + ByteCount; + }); + + return Ptr; +} + +void* +MemoryArena::AllocateAlignedWithOffset(size_t ByteCount, size_t align, size_t offset) +{ + if (ByteCount == 0) + { + return nullptr; + } + + void* Ptr = nullptr; + + m_Lock.WithExclusiveLock([&] { + size_t AlignedOffset = (m_CurrentOffset + (align - 1) + offset) & ~(align - 1); + + if (m_CurrentChunk == nullptr || AlignedOffset + ByteCount > ChunkSize) + { + uint8_t* NewChunk = new uint8_t[ChunkSize]; + if (!NewChunk) + { + return; + } + + m_Chunks.push_back(NewChunk); + m_CurrentChunk = NewChunk; + AlignedOffset = offset; + } + + Ptr = m_CurrentChunk + AlignedOffset; + m_CurrentOffset = AlignedOffset + ByteCount; + }); + + return Ptr; +} + +void* +MemoryArena::Allocate(size_t Size) +{ + if (Size == 0) + { + return nullptr; + } + + void* Ptr = nullptr; + constexpr size_t Alignment = alignof(std::max_align_t); + + m_Lock.WithExclusiveLock([&] { + size_t AlignedOffset = (m_CurrentOffset + Alignment - 1) & ~(Alignment - 1); + + if (m_CurrentChunk == nullptr || AlignedOffset + Size > ChunkSize) + { + uint8_t* NewChunk = new uint8_t[ChunkSize]; + if (!NewChunk) + { + return; + } + + m_Chunks.push_back(NewChunk); + m_CurrentChunk = NewChunk; + AlignedOffset = 0; + } + + Ptr = m_CurrentChunk + AlignedOffset; + m_CurrentOffset = AlignedOffset + Size; + }); + + return Ptr; +} + +const char* +MemoryArena::DuplicateString(std::string_view Str) +{ + const size_t Len = Str.size(); + char* NewStr = static_cast<char*>(Allocate(Len + 1)); + if (NewStr) + { + memcpy(NewStr, Str.data(), Len); + NewStr[Len] = '\0'; + } + return NewStr; +} + +} // namespace zen diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp index fe8fb9a5d..87035554f 100644 --- a/src/zencore/trace.cpp +++ b/src/zencore/trace.cpp @@ -9,6 +9,7 @@ # include <zencore/logging.h> # define TRACE_IMPLEMENT 1 +# undef _WINSOCK_DEPRECATED_NO_WARNINGS # include <zencore/trace.h> # include <zencore/memory/fmalloc.h> diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp index b78991918..4ff79edc7 100644 --- a/src/zencore/zencore.cpp +++ b/src/zencore/zencore.cpp @@ -26,7 +26,6 @@ #include <zencore/parallelwork.h> #include <zencore/process.h> #include <zencore/sha1.h> -#include <zencore/stats.h> #include <zencore/stream.h> #include <zencore/string.h> #include <zencore/thread.h> @@ -267,7 +266,6 @@ zencore_forcelinktests() zen::process_forcelink(); zen::refcount_forcelink(); zen::sha1_forcelink(); - zen::stats_forcelink(); zen::stream_forcelink(); zen::string_forcelink(); zen::thread_forcelink(); diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 2c063d646..f48c22367 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -86,6 +86,9 @@ MapContentTypeToString(HttpContentType ContentType) case HttpContentType::kXML: return "application/xml"sv; + + case HttpContentType::kProtobuf: + return "application/x-protobuf"sv; } } @@ -119,6 +122,7 @@ static constinit uint32_t HashImagePng = HashStringDjb2("image/png"sv); static constinit uint32_t HashIcon = HashStringDjb2("ico"sv); static constinit uint32_t HashImageIcon = HashStringDjb2("image/x-icon"sv); static constinit uint32_t HashXml = HashStringDjb2("application/xml"sv); +static constinit uint32_t HashProtobuf = HashStringDjb2("application/x-protobuf"sv); std::once_flag InitContentTypeLookup; @@ -153,6 +157,7 @@ struct HashedTypeEntry {HashIcon, HttpContentType::kIcon}, {HashImageIcon, HttpContentType::kIcon}, {HashXml, HttpContentType::kXML}, + {HashProtobuf, HttpContentType::kProtobuf}, // clang-format on }; diff --git a/src/zenhttp/include/zenhttp/httptest.h b/src/zenhttp/include/zenhttp/httptest.h index afe71fbce..608462809 100644 --- a/src/zenhttp/include/zenhttp/httptest.h +++ b/src/zenhttp/include/zenhttp/httptest.h @@ -3,8 +3,8 @@ #pragma once #include <zencore/logging.h> -#include <zencore/stats.h> #include <zenhttp/httpserver.h> +#include <zentelemetry/stats.h> #include <atomic> #include <unordered_map> diff --git a/src/zenhttp/xmake.lua b/src/zenhttp/xmake.lua index b6ffbe467..af4064012 100644 --- a/src/zenhttp/xmake.lua +++ b/src/zenhttp/xmake.lua @@ -7,7 +7,7 @@ target('zenhttp') add_files("**.cpp") add_files("servers/httpsys.cpp", {unity_ignored=true}) add_includedirs("include", {public=true}) - add_deps("zencore", "transport-sdk") + add_deps("zencore", "zentelemetry", "transport-sdk") add_packages( "vcpkg::asio", "vcpkg::cpr", diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp index 50cf62274..90af79651 100644 --- a/src/zenserver/diag/logging.cpp +++ b/src/zenserver/diag/logging.cpp @@ -12,6 +12,8 @@ #include <zenutil/logging.h> #include <zenutil/logging/rotatingfilesink.h> +#include "otlphttp.h" + ZEN_THIRD_PARTY_INCLUDES_START #include <spdlog/spdlog.h> ZEN_THIRD_PARTY_INCLUDES_END @@ -73,6 +75,16 @@ InitializeServerLogging(const ZenServerConfig& InOptions) spdlog::apply_logger_env_levels(ZenClientLogger); spdlog::register_logger(ZenClientLogger); + // + +#if ZEN_WITH_OTEL + if (false) + { + auto OtelSink = std::make_shared<zen::logging::OtelHttpProtobufSink>("http://signoz.localdomain:4318"); + zen::logging::Default().SpdLogger->sinks().push_back(std::move(OtelSink)); + } +#endif + FinishInitializeLogging(LogOptions); const zen::Oid ServerSessionId = zen::GetSessionId(); diff --git a/src/zenserver/diag/otlphttp.cpp b/src/zenserver/diag/otlphttp.cpp new file mode 100644 index 000000000..d62ccccb6 --- /dev/null +++ b/src/zenserver/diag/otlphttp.cpp @@ -0,0 +1,83 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "otlphttp.h" + +#include <zencore/config.h> +#include <zencore/process.h> +#include <zencore/session.h> +#include <zencore/system.h> +#include <zentelemetry/otlpencoder.h> +#include <protozero/buffer_string.hpp> +#include <protozero/pbf_builder.hpp> + +#if ZEN_WITH_OTEL + +namespace zen::logging { + +////////////////////////////////////////////////////////////////////////// + +OtelHttpProtobufSink::OtelHttpProtobufSink(const std::string_view& Uri) : m_OtelHttp(Uri) +{ + m_Encoder.AddResourceAttribute("service.name", "zenserver"); + m_Encoder.AddResourceAttribute("service.instance.id", GetSessionIdString()); + m_Encoder.AddResourceAttribute("service.namespace", "zen"); + m_Encoder.AddResourceAttribute("service.version", ZEN_CFG_VERSION); + m_Encoder.AddResourceAttribute("host.name", GetMachineName()); + m_Encoder.AddResourceAttribute("session.id", GetSessionIdString()); + m_Encoder.AddResourceAttribute("process.id", zen::GetCurrentProcessId()); + + m_TraceRecorder = new TraceRecorder(this); + otel::SetTraceRecorder(m_TraceRecorder); +} + +OtelHttpProtobufSink::~OtelHttpProtobufSink() +{ + otel::SetTraceRecorder({}); +} + +void +OtelHttpProtobufSink::RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans) +{ + std::string Data = m_Encoder.FormatOtelTrace(Trace, Spans); + + IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()}; + Payload.SetContentType(ZenContentType::kProtobuf); + + auto Result = m_OtelHttp.Post("/v1/traces", Payload); +} + +void +OtelHttpProtobufSink::TraceRecorder::RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans) +{ + m_Sink->RecordSpans(Trace, Spans); +} + +void +OtelHttpProtobufSink::log(const spdlog::details::log_msg& Msg) +{ + { + std::string Data = m_Encoder.FormatOtelProtobuf(Msg); + + IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()}; + Payload.SetContentType(ZenContentType::kProtobuf); + + auto Result = m_OtelHttp.Post("/v1/logs", Payload); + } + + { + std::string Data = m_Encoder.FormatOtelMetrics(); + + IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()}; + Payload.SetContentType(ZenContentType::kProtobuf); + + auto Result = m_OtelHttp.Post("/v1/metrics", Payload); + } +} +void +OtelHttpProtobufSink::flush() +{ +} + +} // namespace zen::logging + +#endif diff --git a/src/zenserver/diag/otlphttp.h b/src/zenserver/diag/otlphttp.h new file mode 100644 index 000000000..2281bdcc0 --- /dev/null +++ b/src/zenserver/diag/otlphttp.h @@ -0,0 +1,64 @@ + +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <spdlog/sinks/sink.h> +#include <zencore/zencore.h> +#include <zenhttp/httpclient.h> +#include <zentelemetry/otlpencoder.h> +#include <zentelemetry/otlptrace.h> + +#if ZEN_WITH_OTEL + +namespace zen::logging { + +/** + * OTLP/HTTP sink for spdlog + * + * Sends log messages and traces to an OpenTelemetry collector via OTLP over HTTP + */ + +class OtelHttpProtobufSink : public spdlog::sinks::sink +{ +public: + // Note that this URI should be the base URI of the OTLP HTTP endpoint, e.g. + // "http://otel-collector:4318" + OtelHttpProtobufSink(const std::string_view& Uri); + ~OtelHttpProtobufSink(); + + OtelHttpProtobufSink(const OtelHttpProtobufSink&) = delete; + OtelHttpProtobufSink& operator=(const OtelHttpProtobufSink&) = delete; + +private: + virtual void log(const spdlog::details::log_msg& Msg) override; + virtual void flush() override; + virtual void set_pattern(const std::string& pattern) override { ZEN_UNUSED(pattern); } + virtual void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) override { ZEN_UNUSED(sink_formatter); } + + void RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans); + + // This is just a thin wrapper to call back into the sink while participating in + // reference counting from the OTEL trace back-end + class TraceRecorder : public zen::otel::TraceRecorder + { + public: + TraceRecorder(OtelHttpProtobufSink* InSink) : m_Sink(InSink) {} + + private: + TraceRecorder(const TraceRecorder&) = delete; + TraceRecorder& operator=(const TraceRecorder&) = delete; + + virtual void RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans) override; + + OtelHttpProtobufSink* m_Sink; + }; + + HttpClient m_OtelHttp; + OtlpEncoder m_Encoder; + Ref<TraceRecorder> m_TraceRecorder; +}; + +} // namespace zen::logging + +#endif
\ No newline at end of file diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h index 50cb5db12..e10986411 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.h +++ b/src/zenserver/storage/buildstore/httpbuildstore.h @@ -2,10 +2,10 @@ #pragma once -#include <zencore/stats.h> #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> #include <zenhttp/httpstatus.h> +#include <zentelemetry/stats.h> #include <filesystem> diff --git a/src/zenserver/storage/cache/httpstructuredcache.h b/src/zenserver/storage/cache/httpstructuredcache.h index a157148c9..5a795c215 100644 --- a/src/zenserver/storage/cache/httpstructuredcache.h +++ b/src/zenserver/storage/cache/httpstructuredcache.h @@ -2,12 +2,12 @@ #pragma once -#include <zencore/stats.h> #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> #include <zenhttp/httpstatus.h> #include <zenstore/cache/cache.h> #include <zenstore/cache/cacherpc.h> +#include <zentelemetry/stats.h> #include <zenutil/openprocesscache.h> #include <memory> diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h index f0a0bcfa1..f6fe63614 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.h +++ b/src/zenserver/storage/projectstore/httpprojectstore.h @@ -2,11 +2,11 @@ #pragma once -#include <zencore/stats.h> #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> #include <zenhttp/httpstatus.h> #include <zenstore/cidstore.h> +#include <zentelemetry/stats.h> namespace zen { diff --git a/src/zenserver/storage/upstream/upstreamcache.cpp b/src/zenserver/storage/upstream/upstreamcache.cpp index f7ae5f973..6c489c5d3 100644 --- a/src/zenserver/storage/upstream/upstreamcache.cpp +++ b/src/zenserver/storage/upstream/upstreamcache.cpp @@ -9,10 +9,10 @@ #include <zencore/compactbinarypackage.h> #include <zencore/compactbinaryvalidation.h> #include <zencore/fmtutils.h> -#include <zencore/stats.h> #include <zencore/stream.h> #include <zencore/timer.h> #include <zencore/trace.h> +#include <zentelemetry/stats.h> #include <zenhttp/httpclientauth.h> #include <zenhttp/packageformat.h> diff --git a/src/zenserver/storage/upstream/upstreamcache.h b/src/zenserver/storage/upstream/upstreamcache.h index d5d61c8d9..c0c8a7ff9 100644 --- a/src/zenserver/storage/upstream/upstreamcache.h +++ b/src/zenserver/storage/upstream/upstreamcache.h @@ -6,10 +6,10 @@ #include <zencore/compress.h> #include <zencore/iobuffer.h> #include <zencore/iohash.h> -#include <zencore/stats.h> #include <zencore/zencore.h> #include <zenstore/cache/cache.h> #include <zenstore/cache/upstreamcacheclient.h> +#include <zentelemetry/stats.h> #include <atomic> #include <chrono> diff --git a/src/zenserver/storage/workspaces/httpworkspaces.h b/src/zenserver/storage/workspaces/httpworkspaces.h index 89a8e8bdc..888a34b4d 100644 --- a/src/zenserver/storage/workspaces/httpworkspaces.h +++ b/src/zenserver/storage/workspaces/httpworkspaces.h @@ -2,10 +2,10 @@ #pragma once -#include <zencore/stats.h> #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> #include <zenhttp/httpstatus.h> +#include <zentelemetry/stats.h> namespace zen { diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index 57105045d..483bfd5aa 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -7,6 +7,7 @@ target("zenserver") "zennet", "zenremotestore", "zenstore", + "zentelemetry", "zenutil", "zenvfs") add_headerfiles("**.h") @@ -17,6 +18,8 @@ target("zenserver") add_includedirs(".") set_symbols("debug") + add_deps("protozero") + if is_mode("release") then set_optimize("fastest") end diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp index ae1b59dc0..52d5df061 100644 --- a/src/zenstore/cidstore.cpp +++ b/src/zenstore/cidstore.cpp @@ -6,9 +6,9 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> -#include <zencore/stats.h> #include <zencore/string.h> #include <zenstore/scrubcontext.h> +#include <zentelemetry/stats.h> #include "cas.h" diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 10c61681b..1b501e9ae 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -5,10 +5,10 @@ #include "cacheshared.h" #include <zencore/compactbinary.h> -#include <zencore/stats.h> #include <zenstore/accesstime.h> #include <zenstore/blockstore.h> #include <zenstore/caslog.h> +#include <zentelemetry/stats.h> ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h index 1ba469431..75692cfcd 100644 --- a/src/zenstore/include/zenstore/cache/structuredcachestore.h +++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h @@ -4,10 +4,10 @@ #include <zencore/compactbinary.h> #include <zencore/iohash.h> -#include <zencore/stats.h> #include <zenstore/cache/cache.h> #include <zenstore/cache/cachedisklayer.h> #include <zenstore/gc.h> +#include <zentelemetry/stats.h> #include <zenutil/statsreporter.h> #include <atomic> diff --git a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h index 2f3b6b0d7..ff4a8c3f7 100644 --- a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h +++ b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h @@ -6,9 +6,9 @@ #include <zencore/compress.h> #include <zencore/iobuffer.h> #include <zencore/iohash.h> -#include <zencore/stats.h> #include <zencore/zencore.h> #include <zenstore/cache/cache.h> +#include <zentelemetry/stats.h> #include <functional> #include <memory> diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h index 8918b119f..d54062476 100644 --- a/src/zenstore/include/zenstore/cidstore.h +++ b/src/zenstore/include/zenstore/cidstore.h @@ -5,8 +5,8 @@ #include "zenstore.h" #include <zencore/iohash.h> -#include <zencore/stats.h> #include <zenstore/hashkeyset.h> +#include <zentelemetry/stats.h> #include <zenutil/statsreporter.h> #include <filesystem> diff --git a/src/zentelemetry-test/xmake.lua b/src/zentelemetry-test/xmake.lua new file mode 100644 index 000000000..bdc60cee8 --- /dev/null +++ b/src/zentelemetry-test/xmake.lua @@ -0,0 +1,9 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +target("zentelemetry-test") + set_kind("binary") + set_group("tests") + add_headerfiles("**.h") + add_files("*.cpp") + add_deps("zentelemetry") + add_packages("vcpkg::doctest") diff --git a/src/zentelemetry-test/zentelemetry-test.cpp b/src/zentelemetry-test/zentelemetry-test.cpp new file mode 100644 index 000000000..c8b067226 --- /dev/null +++ b/src/zentelemetry-test/zentelemetry-test.cpp @@ -0,0 +1,42 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/filesystem.h> +#include <zencore/logging.h> +#include <zencore/trace.h> +#include <zentelemetry/zentelemetry.h> + +#include <zencore/memory/newdelete.h> + +#if ZEN_WITH_TESTS +# define ZEN_TEST_WITH_RUNNER 1 +# include <zencore/testing.h> +# include <zencore/process.h> +#endif + +int +main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) +{ +#if ZEN_WITH_TESTS + zen::zentelemetry_forcelinktests(); + +# if ZEN_PLATFORM_LINUX + zen::IgnoreChildSignals(); +# endif + +# if ZEN_WITH_TRACE + zen::TraceInit("zenstore-test"); + zen::TraceOptions TraceCommandlineOptions; + if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) + { + TraceConfigure(TraceCommandlineOptions); + } +# endif // ZEN_WITH_TRACE + + zen::logging::InitializeLogging(); + zen::MaximizeOpenFileCount(); + + return ZEN_RUN_TESTS(argc, argv); +#else + return 0; +#endif +} diff --git a/src/zentelemetry/include/zentelemetry/otlpencoder.h b/src/zentelemetry/include/zentelemetry/otlpencoder.h new file mode 100644 index 000000000..ed6665781 --- /dev/null +++ b/src/zentelemetry/include/zentelemetry/otlpencoder.h @@ -0,0 +1,66 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/thread.h> +#include <zentelemetry/otlptrace.h> +#include <string> +#include <string_view> +#include <unordered_map> + +#if ZEN_WITH_OTEL + +# include <protozero/pbf_builder.hpp> +# include <protozero/types.hpp> + +namespace spdlog { namespace details { + struct log_msg; +}} // namespace spdlog::details + +namespace zen::otel { +enum class Resource : protozero::pbf_tag_type; +} + +namespace zen { + +/** + * Encoder for OpenTelemetry OTLP Protobuf format + * + * Designed to encode log messages and metrics into the OTLP Protobuf format + * for transmission to an OpenTelemetry collector or compatible backend. + * + * Currently, only log messages and basic resource attributes are supported. + * + * Some initial support for metrics encoding is also included. + * + * Transmission is expected to be handled externally, e.g. via HTTP client. + * + */ + +class OtlpEncoder +{ +public: + OtlpEncoder(); + ~OtlpEncoder(); + + void AddResourceAttribute(const std::string_view& Key, const std::string_view& Value); + void AddResourceAttribute(const std::string_view& Key, int64_t Value); + + std::string FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const; + std::string FormatOtelMetrics() const; + std::string FormatOtelTrace(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans) const; + + OtlpEncoder& operator=(const OtlpEncoder&) = delete; + OtlpEncoder(const OtlpEncoder&) = delete; + +private: + mutable RwLock m_ResourceLock; + std::unordered_map<std::string, std::string> m_ResourceAttributes; + std::unordered_map<std::string, int64_t> m_ResourceIntAttributes; + + void AppendResourceAttributes(protozero::pbf_builder<otel::Resource>& Res) const; +}; + +} // namespace zen + +#endif diff --git a/src/zentelemetry/include/zentelemetry/otlptrace.h b/src/zentelemetry/include/zentelemetry/otlptrace.h new file mode 100644 index 000000000..ebecff91a --- /dev/null +++ b/src/zentelemetry/include/zentelemetry/otlptrace.h @@ -0,0 +1,268 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenbase/refcount.h> +#include <zencore/uid.h> + +#include <span> +#include <string> + +#define ZEN_WITH_OTEL 1 + +#if ZEN_WITH_OTEL +# define ZEN_CONCAT(a, b) a##b +# define IMPL_ZEN_OTEL_SPAN(Name, Line) ::zen::otel::ScopedSpan ZEN_CONCAT(Span$, Line)(Name); +# define ZEN_OTEL_SPAN(Name) IMPL_ZEN_OTEL_SPAN(Name, __LINE__) +#else +# define ZEN_OTEL_SPAN(Name) +#endif + +namespace zen { +class MemoryArena; +} + +#if ZEN_WITH_OTEL + +namespace zen::otel { + +using AttributeList = std::span<std::pair<std::string, std::string>>; + +class Tracer; + +Tracer& GetTracer(); + +// OLTP Span ID + +struct SpanId +{ + constexpr static size_t kSize = 8; + + inline SpanId() : Id{0} {} + explicit SpanId(const std::span<const uint8_t, 8> Bytes) noexcept { std::copy(Bytes.begin(), Bytes.end(), Id); } + explicit SpanId(const Oid& InId) noexcept { memcpy(Id, reinterpret_cast<const uint8_t*>(InId.OidBits), sizeof Id); } + + auto operator<=>(const SpanId& Rhs) const = default; + inline operator bool() const { return *this != SpanId(); } + + static SpanId NewSpanId(); + + const char* GetData() const { return reinterpret_cast<const char*>(Id); } + +private: + uint8_t Id[kSize]; +}; + +// OLTP Trace ID + +struct TraceId +{ + constexpr static size_t kSize = 16; + + std::span<const uint8_t> GetBytes() const { return std::span<const uint8_t>(Id, kSize); } + + inline TraceId() noexcept : Id{0} {} + explicit TraceId(const std::span<const uint8_t, kSize> Bytes) noexcept { std::copy(Bytes.begin(), Bytes.end(), Id); } + + auto operator<=>(const TraceId& Rhs) const = default; + inline operator bool() const { return *this != TraceId(); } + + static TraceId NewTraceId(); + + inline const char* GetData() const { return reinterpret_cast<const char*>(Id); } + +private: + uint8_t Id[kSize]; +}; + +struct AttributePair +{ + const char* Key = nullptr; + union + { + const char* StringValue; + uint64_t NumericValue; + } Value; + + uint64_t Flags = 0; + + enum + { + kFlags_IsNumeric = 1 << 0, + kFlags_IsString = 1 << 1 + }; + + bool IsNumeric() const { return (Flags & kFlags_IsNumeric) != 0; } + uint64_t GetNumericValue() const { return Value.NumericValue; } + void SetNumericValue(uint64_t InValue) + { + Value.NumericValue = InValue; + Flags |= kFlags_IsNumeric; + } + + bool IsString() const { return (Flags & kFlags_IsString) != 0; } + const char* GetStringValue() const { return Value.StringValue; } + void SetStringValue(const char* InValue) + { + Value.StringValue = InValue; + Flags |= kFlags_IsString; + } + + AttributePair* Next = nullptr; +}; + +struct Event +{ + Event* NextEvent = nullptr; + uint64_t Timestamp = 0; + const char* Name = nullptr; + AttributePair* Attributes = nullptr; +}; + +/** Span within a trace + * + * A span represents a single operation within a trace. Spans can be nested + * to form a trace tree. + * + */ + +struct Span final : public TRefCounted<Span> +{ + Span& operator=(const Span&) = delete; + Span(const Span&) = delete; + + void AddEvent(std::string_view Name); + void AddEvent(std::string_view Name, uint64_t Timestamp); + void AddEvent(std::string_view Name, const AttributeList& Attributes); + + void AddAttribute(std::string_view Key, std::string_view Value); + void AddAttribute(std::string_view Key, uint64_t Value); + void AddAttributes(const AttributeList& Attributes); + + void End(); + + enum class Kind : uint8_t // This must match otel::SpanKind + { + kInternal = 1, + kServer = 2, + kClient = 3, + kProducer = 4, + kConsumer = 5 + }; + + Kind GetKind() const { return m_Kind; } + void SetKind(Kind InKind) { m_Kind = InKind; } + + SpanId GetSpanId() const { return m_SpanId; } + const char* GetName() const { return m_Name; } + uint64_t GetStartTime() const { return m_StartTime; } + uint64_t GetEndTime() const { return m_EndTime; } + const Span* GetParentSpan() const { return m_ParentSpan; } + const AttributePair* GetAttributes() const { return m_Attributes; } + const Event* GetEvents() const { return m_Events; } + + // This is used to get the current span/trace ID for logging purposes + static SpanId GetCurrentSpanId(TraceId& OutTraceId); + static Span* GetCurrentSpan(); + +protected: + void* operator new(size_t Size) = delete; + void* operator new(size_t Size, MemoryArena& Arena); + void operator delete(void* Ptr) = delete; + void operator delete(void* Ptr, MemoryArena& Arena); + + friend class ScopedSpan; + friend class Tracer; + friend class TRefCounted<Span>; + + void DeleteThis() noexcept { End(); } + +private: + // Note: for now there's no locking on the span. To support spans across + // threads, we'll need to add locking around event addition and ending + // and linking of child spans etc + + MemoryArena& m_Arena; + Span* m_NextSibling = nullptr; + Span* m_FirstChild = nullptr; + Span* m_ParentSpan = nullptr; + Span* m_SpanChain = nullptr; + Event* m_Events = nullptr; + AttributePair* m_Attributes = nullptr; + SpanId m_SpanId; + const char* m_Name = nullptr; + uint64_t m_StartTime = 0; + uint64_t m_EndTime = 0; + Kind m_Kind = Kind::kInternal; + bool m_Ended = false; + + Span(MemoryArena& Arena, std::string_view Name, Span* SpanChain); + ~Span(); +}; + +/** Scoped span helper + * + * Automatically ends the span when it goes out of scope + */ + +class ScopedSpan final +{ +public: + ScopedSpan(std::string_view Name); + ScopedSpan() = delete; + ~ScopedSpan(); + + ScopedSpan& operator=(ScopedSpan&& Rhs) = default; + ScopedSpan& operator=(const ScopedSpan& Rhs) = default; + ScopedSpan(ScopedSpan&& Rhs) = default; + ScopedSpan(const ScopedSpan& Rhs) = default; + + Span* operator->() const { return m_Span.Get(); } + +private: + ScopedSpan(Span* InSpan, Tracer* InTracer); + + Ref<Tracer> m_Tracer; // This needs to precede the span ref to ensure proper destruction order + Ref<Span> m_Span; + + friend class Tracer; +}; + +/** + * Tracer for creating spans and managing trace state. + * + * This represents a single trace context identified by a trace ID, + * and is at the root of all spans created within that trace. + * + */ + +class Tracer final : public RefCounted +{ +public: + static ScopedSpan CreateSpan(std::string_view Name); + +private: + struct Impl; + Impl* m_Impl; + + Tracer(); + ~Tracer(); + + static Tracer* GetTracer(); + + friend class ScopedSpan; +}; + +class TraceRecorder : public RefCounted +{ +public: + virtual void RecordSpans(TraceId Trace, std::span<const Span*> Spans) = 0; +}; + +void SetTraceRecorder(Ref<TraceRecorder> Recorder); +bool IsRecording(); + +void otlptrace_forcelink(); + +} // namespace zen::otel +#endif diff --git a/src/zencore/include/zencore/stats.h b/src/zentelemetry/include/zentelemetry/stats.h index f232cf2f4..3e67bac1c 100644 --- a/src/zencore/include/zencore/stats.h +++ b/src/zentelemetry/include/zentelemetry/stats.h @@ -2,7 +2,7 @@ #pragma once -#include "zencore.h" +#include "zentelemetry.h" #include <zenbase/concepts.h> diff --git a/src/zentelemetry/include/zentelemetry/zentelemetry.h b/src/zentelemetry/include/zentelemetry/zentelemetry.h new file mode 100644 index 000000000..fc811974b --- /dev/null +++ b/src/zentelemetry/include/zentelemetry/zentelemetry.h @@ -0,0 +1,9 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +namespace zen { + +void zentelemetry_forcelinktests(); + +} diff --git a/src/zentelemetry/otellogprotozero.h b/src/zentelemetry/otellogprotozero.h new file mode 100644 index 000000000..1e2e85d2d --- /dev/null +++ b/src/zentelemetry/otellogprotozero.h @@ -0,0 +1,287 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "otelprotozero.h" + +////////////////////////////////////////////////////////////////////////// +// +// OTEL .proto definitions, for reference +// + +#if 0 + +// clang-format off + +///////////////////////////////////////////////////////////////////////////////////// +// logs/v1 + +// LogsData represents the logs data that can be stored in a persistent storage, +// OR can be embedded by other protocols that transfer OTLP logs data but do not +// implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message LogsData { + // An array of ResourceLogs. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceLogs resource_logs = 1; +} + +// A collection of ScopeLogs from a Resource. +message ResourceLogs { + reserved 1000; + + // The resource for the logs in this message. + // If this field is not set then resource info is unknown. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeLogs that originate from a resource. + repeated ScopeLogs scope_logs = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_logs" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Logs produced by a Scope. +message ScopeLogs { + // The instrumentation scope information for the logs in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of log records. + repeated LogRecord log_records = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the log data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all logs in the "logs" field. + string schema_url = 3; +} + +// Possible values for LogRecord.SeverityNumber. +enum SeverityNumber { + // UNSPECIFIED is the default SeverityNumber, it MUST NOT be used. + SEVERITY_NUMBER_UNSPECIFIED = 0; + SEVERITY_NUMBER_TRACE = 1; + SEVERITY_NUMBER_TRACE2 = 2; + SEVERITY_NUMBER_TRACE3 = 3; + SEVERITY_NUMBER_TRACE4 = 4; + SEVERITY_NUMBER_DEBUG = 5; + SEVERITY_NUMBER_DEBUG2 = 6; + SEVERITY_NUMBER_DEBUG3 = 7; + SEVERITY_NUMBER_DEBUG4 = 8; + SEVERITY_NUMBER_INFO = 9; + SEVERITY_NUMBER_INFO2 = 10; + SEVERITY_NUMBER_INFO3 = 11; + SEVERITY_NUMBER_INFO4 = 12; + SEVERITY_NUMBER_WARN = 13; + SEVERITY_NUMBER_WARN2 = 14; + SEVERITY_NUMBER_WARN3 = 15; + SEVERITY_NUMBER_WARN4 = 16; + SEVERITY_NUMBER_ERROR = 17; + SEVERITY_NUMBER_ERROR2 = 18; + SEVERITY_NUMBER_ERROR3 = 19; + SEVERITY_NUMBER_ERROR4 = 20; + SEVERITY_NUMBER_FATAL = 21; + SEVERITY_NUMBER_FATAL2 = 22; + SEVERITY_NUMBER_FATAL3 = 23; + SEVERITY_NUMBER_FATAL4 = 24; +} + +// LogRecordFlags represents constants used to interpret the +// LogRecord.flags field, which is protobuf 'fixed32' type and is to +// be used as bit-fields. Each non-zero value defined in this enum is +// a bit-mask. To extract the bit-field, for example, use an +// expression like: +// +// (logRecord.flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK) +// +enum LogRecordFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + LOG_RECORD_FLAGS_DO_NOT_USE = 0; + + // Bits 0-7 are used for trace flags. + LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; + + // Bits 8-31 are reserved for future use. +} + +// A log record according to OpenTelemetry Log Data Model: +// https://github.com/open-telemetry/oteps/blob/main/text/logs/0097-log-data-model.md +message LogRecord { + reserved 4; + + // time_unix_nano is the time when the event occurred. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // Value of 0 indicates unknown or missing timestamp. + fixed64 time_unix_nano = 1; + + // Time when the event was observed by the collection system. + // For events that originate in OpenTelemetry (e.g. using OpenTelemetry Logging SDK) + // this timestamp is typically set at the generation time and is equal to Timestamp. + // For events originating externally and collected by OpenTelemetry (e.g. using + // Collector) this is the time when OpenTelemetry's code observed the event measured + // by the clock of the OpenTelemetry code. This field MUST be set once the event is + // observed by OpenTelemetry. + // + // For converting OpenTelemetry log data to formats that support only one timestamp or + // when receiving OpenTelemetry log data by recipients that support only one timestamp + // internally the following logic is recommended: + // - Use time_unix_nano if it is present, otherwise use observed_time_unix_nano. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // Value of 0 indicates unknown or missing timestamp. + fixed64 observed_time_unix_nano = 11; + + // Numerical value of the severity, normalized to values described in Log Data Model. + // [Optional]. + SeverityNumber severity_number = 2; + + // The severity text (also known as log level). The original string representation as + // it is known at the source. [Optional]. + string severity_text = 3; + + // A value containing the body of the log record. Can be for example a human-readable + // string message (including multi-line) describing the event in a free form or it can + // be a structured data composed of arrays and maps of other values. [Optional]. + opentelemetry.proto.common.v1.AnyValue body = 5; + + // Additional attributes that describe the specific event occurrence. [Optional]. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 6; + uint32 dropped_attributes_count = 7; + + // Flags, a bit field. 8 least significant bits are the trace flags as + // defined in W3C Trace Context specification. 24 most significant bits are reserved + // and must be set to 0. Readers must not assume that 24 most significant bits + // will be zero and must correctly mask the bits when reading 8-bit trace flag (use + // flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK). [Optional]. + fixed32 flags = 8; + + // A unique identifier for a trace. All logs from the same trace share + // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is optional. + // + // The receivers SHOULD assume that the log record is not associated with a + // trace if any of the following is true: + // - the field is not present, + // - the field contains an invalid value. + bytes trace_id = 9; + + // A unique identifier for a span within a trace, assigned when the span + // is created. The ID is an 8-byte array. An ID with all zeroes OR of length + // other than 8 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is optional. If the sender specifies a valid span_id then it SHOULD also + // specify a valid trace_id. + // + // The receivers SHOULD assume that the log record is not associated with a + // span if any of the following is true: + // - the field is not present, + // - the field contains an invalid value. + bytes span_id = 10; + + // A unique identifier of event category/type. + // All events with the same event_name are expected to conform to the same + // schema for both their attributes and their body. + // + // Recommended to be fully qualified and short (no longer than 256 characters). + // + // Presence of event_name on the log record identifies this record + // as an event. + // + // [Optional]. + string event_name = 12; +} +// clang-format on +#endif + +namespace zen::otel { + +// Possible values for LogRecord.SeverityNumber. +enum SeverityNumber +{ + // UNSPECIFIED is the default SeverityNumber, it MUST NOT be used. + SEVERITY_NUMBER_UNSPECIFIED = 0, + SEVERITY_NUMBER_TRACE = 1, + SEVERITY_NUMBER_TRACE2 = 2, + SEVERITY_NUMBER_TRACE3 = 3, + SEVERITY_NUMBER_TRACE4 = 4, + SEVERITY_NUMBER_DEBUG = 5, + SEVERITY_NUMBER_DEBUG2 = 6, + SEVERITY_NUMBER_DEBUG3 = 7, + SEVERITY_NUMBER_DEBUG4 = 8, + SEVERITY_NUMBER_INFO = 9, + SEVERITY_NUMBER_INFO2 = 10, + SEVERITY_NUMBER_INFO3 = 11, + SEVERITY_NUMBER_INFO4 = 12, + SEVERITY_NUMBER_WARN = 13, + SEVERITY_NUMBER_WARN2 = 14, + SEVERITY_NUMBER_WARN3 = 15, + SEVERITY_NUMBER_WARN4 = 16, + SEVERITY_NUMBER_ERROR = 17, + SEVERITY_NUMBER_ERROR2 = 18, + SEVERITY_NUMBER_ERROR3 = 19, + SEVERITY_NUMBER_ERROR4 = 20, + SEVERITY_NUMBER_FATAL = 21, + SEVERITY_NUMBER_FATAL2 = 22, + SEVERITY_NUMBER_FATAL3 = 23, + SEVERITY_NUMBER_FATAL4 = 24 +}; + +enum class LogRecord : protozero::pbf_tag_type +{ + required_fixed64_time_unix_nano = 1, + optional_SeverityNumber_severity_number = 2, + optional_string_severity_text = 3, + optional_anyvalue_body = 5, + optional_repeated_kv_attributes = 6, + optional_uint32_dropped_attributes_count = 7, + optional_fixed32_flags = 8, + optional_bytes_trace_id = 9, + optional_bytes_span_id = 10, + optional_fixed64_observed_time_unix_nano = 11, + optional_event_name = 12, +}; + +enum class ScopeLogs : protozero::pbf_tag_type +{ + required_InstrumentationScope_scope = 1, + required_repeated_LogRecord_log_records = 2, + optional_string_schema_url = 3 +}; + +enum class ResourceLogs : protozero::pbf_tag_type +{ + optional_Resource_resource = 1, + required_repeated_ScopeLogs_scope_logs = 2, + optional_string_schema_url = 3 +}; + +enum class LogsData : protozero::pbf_tag_type +{ + required_repeated_ResourceLogs_resource_logs = 1 +}; + +} // namespace zen::otel diff --git a/src/zentelemetry/otelmetricsprotozero.h b/src/zentelemetry/otelmetricsprotozero.h new file mode 100644 index 000000000..12bae0d75 --- /dev/null +++ b/src/zentelemetry/otelmetricsprotozero.h @@ -0,0 +1,953 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "otelprotozero.h" + +////////////////////////////////////////////////////////////////////////// +// +// OTEL .proto definitions, for reference +// + +#if 0 + +// clang-format off + +////////////////////////////////////////////////////////////////////////// +// metrics/v1/metrics.proto +// + +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.metrics.v1; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Metrics.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.metrics.v1"; +option java_outer_classname = "MetricsProto"; +option go_package = "go.opentelemetry.io/proto/otlp/metrics/v1"; + +// MetricsData represents the metrics data that can be stored in a persistent +// storage, OR can be embedded by other protocols that transfer OTLP metrics +// data but do not implement the OTLP protocol. +// +// MetricsData +// └─── ResourceMetrics +// ├── Resource +// ├── SchemaURL +// └── ScopeMetrics +// ├── Scope +// ├── SchemaURL +// └── Metric +// ├── Name +// ├── Description +// ├── Unit +// └── data +// ├── Gauge +// ├── Sum +// ├── Histogram +// ├── ExponentialHistogram +// └── Summary +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message MetricsData { + // An array of ResourceMetrics. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceMetrics resource_metrics = 1; +} + +// A collection of ScopeMetrics from a Resource. +message ResourceMetrics { + reserved 1000; + + // The resource for the metrics in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of metrics that originate from a resource. + repeated ScopeMetrics scope_metrics = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_metrics" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Metrics produced by an Scope. +message ScopeMetrics { + // The instrumentation scope information for the metrics in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of metrics that originate from an instrumentation library. + repeated Metric metrics = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the metric data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all metrics in the "metrics" field. + string schema_url = 3; +} + +// Defines a Metric which has one or more timeseries. The following is a +// brief summary of the Metric data model. For more details, see: +// +// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md +// +// The data model and relation between entities is shown in the +// diagram below. Here, "DataPoint" is the term used to refer to any +// one of the specific data point value types, and "points" is the term used +// to refer to any one of the lists of points contained in the Metric. +// +// - Metric is composed of a metadata and data. +// - Metadata part contains a name, description, unit. +// - Data is one of the possible types (Sum, Gauge, Histogram, Summary). +// - DataPoint contains timestamps, attributes, and one of the possible value type +// fields. +// +// Metric +// +------------+ +// |name | +// |description | +// |unit | +------------------------------------+ +// |data |---> |Gauge, Sum, Histogram, Summary, ... | +// +------------+ +------------------------------------+ +// +// Data [One of Gauge, Sum, Histogram, Summary, ...] +// +-----------+ +// |... | // Metadata about the Data. +// |points |--+ +// +-----------+ | +// | +---------------------------+ +// | |DataPoint 1 | +// v |+------+------+ +------+ | +// +-----+ ||label |label |...|label | | +// | 1 |-->||value1|value2|...|valueN| | +// +-----+ |+------+------+ +------+ | +// | . | |+-----+ | +// | . | ||value| | +// | . | |+-----+ | +// | . | +---------------------------+ +// | . | . +// | . | . +// | . | . +// | . | +---------------------------+ +// | . | |DataPoint M | +// +-----+ |+------+------+ +------+ | +// | M |-->||label |label |...|label | | +// +-----+ ||value1|value2|...|valueN| | +// |+------+------+ +------+ | +// |+-----+ | +// ||value| | +// |+-----+ | +// +---------------------------+ +// +// Each distinct type of DataPoint represents the output of a specific +// aggregation function, the result of applying the DataPoint's +// associated function of to one or more measurements. +// +// All DataPoint types have three common fields: +// - Attributes includes key-value pairs associated with the data point +// - TimeUnixNano is required, set to the end time of the aggregation +// - StartTimeUnixNano is optional, but strongly encouraged for DataPoints +// having an AggregationTemporality field, as discussed below. +// +// Both TimeUnixNano and StartTimeUnixNano values are expressed as +// UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. +// +// # TimeUnixNano +// +// This field is required, having consistent interpretation across +// DataPoint types. TimeUnixNano is the moment corresponding to when +// the data point's aggregate value was captured. +// +// Data points with the 0 value for TimeUnixNano SHOULD be rejected +// by consumers. +// +// # StartTimeUnixNano +// +// StartTimeUnixNano in general allows detecting when a sequence of +// observations is unbroken. This field indicates to consumers the +// start time for points with cumulative and delta +// AggregationTemporality, and it should be included whenever possible +// to support correct rate calculation. Although it may be omitted +// when the start time is truly unknown, setting StartTimeUnixNano is +// strongly encouraged. +message Metric { + reserved 4, 6, 8; + + // The name of the metric. + string name = 1; + + // A description of the metric, which can be used in documentation. + string description = 2; + + // The unit in which the metric value is reported. Follows the format + // described by https://unitsofmeasure.org/ucum.html. + string unit = 3; + + // Data determines the aggregation type (if any) of the metric, what is the + // reported value type for the data points, as well as the relatationship to + // the time interval over which they are reported. + oneof data { + Gauge gauge = 5; + Sum sum = 7; + Histogram histogram = 9; + ExponentialHistogram exponential_histogram = 10; + Summary summary = 11; + } + + // Additional metadata attributes that describe the metric. [Optional]. + // Attributes are non-identifying. + // Consumers SHOULD NOT need to be aware of these attributes. + // These attributes MAY be used to encode information allowing + // for lossless roundtrip translation to / from another data model. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue metadata = 12; +} + +// Gauge represents the type of a scalar metric that always exports the +// "current value" for every data point. It should be used for an "unknown" +// aggregation. +// +// A Gauge does not support different aggregation temporalities. Given the +// aggregation is unknown, points cannot be combined using the same +// aggregation, regardless of aggregation temporalities. Therefore, +// AggregationTemporality is not included. Consequently, this also means +// "StartTimeUnixNano" is ignored for all data points. +message Gauge { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated NumberDataPoint data_points = 1; +} + +// Sum represents the type of a scalar metric that is calculated as a sum of all +// reported measurements over a time interval. +message Sum { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated NumberDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; + + // Represents whether the sum is monotonic. + bool is_monotonic = 3; +} + +// Histogram represents the type of a metric that is calculated by aggregating +// as a Histogram of all reported measurements over a time interval. +message Histogram { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated HistogramDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// ExponentialHistogram represents the type of a metric that is calculated by aggregating +// as a ExponentialHistogram of all reported double measurements over a time interval. +message ExponentialHistogram { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated ExponentialHistogramDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// Summary metric data are used to convey quantile summaries, +// a Prometheus (see: https://prometheus.io/docs/concepts/metric_types/#summary) +// and OpenMetrics (see: https://github.com/prometheus/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45) +// data type. These data points cannot always be merged in a meaningful way. +// While they can be useful in some applications, histogram data points are +// recommended for new applications. +// Summary metrics do not have an aggregation temporality field. This is +// because the count and sum fields of a SummaryDataPoint are assumed to be +// cumulative values. +message Summary { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated SummaryDataPoint data_points = 1; +} + +// AggregationTemporality defines how a metric aggregator reports aggregated +// values. It describes how those values relate to the time interval over +// which they are aggregated. +enum AggregationTemporality { + // UNSPECIFIED is the default AggregationTemporality, it MUST not be used. + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + + // DELTA is an AggregationTemporality for a metric aggregator which reports + // changes since last report time. Successive metrics contain aggregation of + // values from continuous and non-overlapping intervals. + // + // The values for a DELTA metric are based only on the time interval + // associated with one measurement cycle. There is no dependency on + // previous measurements like is the case for CUMULATIVE metrics. + // + // For example, consider a system measuring the number of requests that + // it receives and reports the sum of these requests every second as a + // DELTA metric: + // + // 1. The system starts receiving at time=t_0. + // 2. A request is received, the system measures 1 request. + // 3. A request is received, the system measures 1 request. + // 4. A request is received, the system measures 1 request. + // 5. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0 to + // t_0+1 with a value of 3. + // 6. A request is received, the system measures 1 request. + // 7. A request is received, the system measures 1 request. + // 8. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0+1 to + // t_0+2 with a value of 2. + AGGREGATION_TEMPORALITY_DELTA = 1; + + // CUMULATIVE is an AggregationTemporality for a metric aggregator which + // reports changes since a fixed start time. This means that current values + // of a CUMULATIVE metric depend on all previous measurements since the + // start time. Because of this, the sender is required to retain this state + // in some form. If this state is lost or invalidated, the CUMULATIVE metric + // values MUST be reset and a new fixed start time following the last + // reported measurement time sent MUST be used. + // + // For example, consider a system measuring the number of requests that + // it receives and reports the sum of these requests every second as a + // CUMULATIVE metric: + // + // 1. The system starts receiving at time=t_0. + // 2. A request is received, the system measures 1 request. + // 3. A request is received, the system measures 1 request. + // 4. A request is received, the system measures 1 request. + // 5. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0 to + // t_0+1 with a value of 3. + // 6. A request is received, the system measures 1 request. + // 7. A request is received, the system measures 1 request. + // 8. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0 to + // t_0+2 with a value of 5. + // 9. The system experiences a fault and loses state. + // 10. The system recovers and resumes receiving at time=t_1. + // 11. A request is received, the system measures 1 request. + // 12. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_1 to + // t_0+1 with a value of 1. + // + // Note: Even though, when reporting changes since last report time, using + // CUMULATIVE is valid, it is not recommended. This may cause problems for + // systems that do not use start_time to determine when the aggregation + // value was reset (e.g. Prometheus). + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + +// DataPointFlags is defined as a protobuf 'uint32' type and is to be used as a +// bit-field representing 32 distinct boolean flags. Each flag defined in this +// enum is a bit-mask. To test the presence of a single flag in the flags of +// a data point, for example, use an expression like: +// +// (point.flags & DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) == DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK +// +enum DataPointFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + DATA_POINT_FLAGS_DO_NOT_USE = 0; + + // This DataPoint is valid but has no recorded value. This value + // SHOULD be used to reflect explicitly missing data in a series, as + // for an equivalent to the Prometheus "staleness marker". + DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK = 1; + + // Bits 2-31 are reserved for future use. +} + +// NumberDataPoint is a single data point in a timeseries that describes the +// time-varying scalar value of a metric. +message NumberDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list may be empty (may contain 0 elements). + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // + // The attribute values SHOULD NOT contain empty values. + // The attribute values SHOULD NOT contain bytes values. + // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values, + // double values. + // The attribute values SHOULD NOT contain kvlist values. + // The behavior of software that receives attributes containing such values can be unpredictable. + // These restrictions can change in a minor release. + // The restrictions take origin from the OpenTelemetry specification: + // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // The value itself. A point is considered invalid when one of the recognized + // value fields is not present inside this oneof. + oneof value { + double as_double = 4; + sfixed64 as_int = 6; + } + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 5; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 8; +} + +// HistogramDataPoint is a single data point in a timeseries that describes the +// time-varying values of a Histogram. A Histogram contains summary statistics +// for a population of values, it may optionally contain the distribution of +// those values across a set of buckets. +// +// If the histogram contains the distribution of values, then both +// "explicit_bounds" and "bucket counts" fields must be defined. +// If the histogram does not contain the distribution of values, then both +// "explicit_bounds" and "bucket_counts" must be omitted and only "count" and +// "sum" are known. +message HistogramDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list may be empty (may contain 0 elements). + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // + // The attribute values SHOULD NOT contain empty values. + // The attribute values SHOULD NOT contain bytes values. + // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values, + // double values. + // The attribute values SHOULD NOT contain kvlist values. + // The behavior of software that receives attributes containing such values can be unpredictable. + // These restrictions can change in a minor release. + // The restrictions take origin from the OpenTelemetry specification: + // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // count is the number of values in the population. Must be non-negative. This + // value must be equal to the sum of the "count" fields in buckets if a + // histogram is provided. + fixed64 count = 4; + + // sum of the values in the population. If count is zero then this field + // must be zero. + // + // Note: Sum should only be filled out when measuring non-negative discrete + // events, and is assumed to be monotonic over the values of these events. + // Negative events *can* be recorded, but sum should not be filled out when + // doing so. This is specifically to enforce compatibility w/ OpenMetrics, + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram + optional double sum = 5; + + // bucket_counts is an optional field contains the count values of histogram + // for each bucket. + // + // The sum of the bucket_counts must equal the value in the count field. + // + // The number of elements in bucket_counts array must be by one greater than + // the number of elements in explicit_bounds array. The exception to this rule + // is when the length of bucket_counts is 0, then the length of explicit_bounds + // must also be 0. + repeated fixed64 bucket_counts = 6; + + // explicit_bounds specifies buckets with explicitly defined bounds for values. + // + // The boundaries for bucket at index i are: + // + // (-infinity, explicit_bounds[i]] for i == 0 + // (explicit_bounds[i-1], explicit_bounds[i]] for 0 < i < size(explicit_bounds) + // (explicit_bounds[i-1], +infinity) for i == size(explicit_bounds) + // + // The values in the explicit_bounds array must be strictly increasing. + // + // Histogram buckets are inclusive of their upper boundary, except the last + // bucket where the boundary is at infinity. This format is intentionally + // compatible with the OpenMetrics histogram definition. + // + // If bucket_counts length is 0 then explicit_bounds length must also be 0, + // otherwise the data point is invalid. + repeated double explicit_bounds = 7; + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 8; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 10; + + // min is the minimum value over (start_time, end_time]. + optional double min = 11; + + // max is the maximum value over (start_time, end_time]. + optional double max = 12; +} + +// ExponentialHistogramDataPoint is a single data point in a timeseries that describes the +// time-varying values of a ExponentialHistogram of double values. A ExponentialHistogram contains +// summary statistics for a population of values, it may optionally contain the +// distribution of those values across a set of buckets. +// +message ExponentialHistogramDataPoint { + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list may be empty (may contain 0 elements). + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // + // The attribute values SHOULD NOT contain empty values. + // The attribute values SHOULD NOT contain bytes values. + // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values, + // double values. + // The attribute values SHOULD NOT contain kvlist values. + // The behavior of software that receives attributes containing such values can be unpredictable. + // These restrictions can change in a minor release. + // The restrictions take origin from the OpenTelemetry specification: + // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // The number of values in the population. Must be + // non-negative. This value must be equal to the sum of the "bucket_counts" + // values in the positive and negative Buckets plus the "zero_count" field. + fixed64 count = 4; + + // The sum of the values in the population. If count is zero then this field + // must be zero. + // + // Note: Sum should only be filled out when measuring non-negative discrete + // events, and is assumed to be monotonic over the values of these events. + // Negative events *can* be recorded, but sum should not be filled out when + // doing so. This is specifically to enforce compatibility w/ OpenMetrics, + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram + optional double sum = 5; + + // scale describes the resolution of the histogram. Boundaries are + // located at powers of the base, where: + // + // base = (2^(2^-scale)) + // + // The histogram bucket identified by `index`, a signed integer, + // contains values that are greater than (base^index) and + // less than or equal to (base^(index+1)). + // + // The positive and negative ranges of the histogram are expressed + // separately. Negative values are mapped by their absolute value + // into the negative range using the same scale as the positive range. + // + // scale is not restricted by the protocol, as the permissible + // values depend on the range of the data. + sint32 scale = 6; + + // The count of values that are either exactly zero or + // within the region considered zero by the instrumentation at the + // tolerated degree of precision. This bucket stores values that + // cannot be expressed using the standard exponential formula as + // well as values that have been rounded to zero. + // + // Implementations MAY consider the zero bucket to have probability + // mass equal to (zero_count / count). + fixed64 zero_count = 7; + + // positive carries the positive range of exponential bucket counts. + Buckets positive = 8; + + // negative carries the negative range of exponential bucket counts. + Buckets negative = 9; + + // Buckets are a set of bucket counts, encoded in a contiguous array + // of counts. + message Buckets { + // The bucket index of the first entry in the bucket_counts array. + // + // Note: This uses a varint encoding as a simple form of compression. + sint32 offset = 1; + + // An array of count values, where bucket_counts[i] carries + // the count of the bucket at index (offset+i). bucket_counts[i] is the count + // of values greater than base^(offset+i) and less than or equal to + // base^(offset+i+1). + // + // Note: By contrast, the explicit HistogramDataPoint uses + // fixed64. This field is expected to have many buckets, + // especially zeros, so uint64 has been selected to ensure + // varint encoding. + repeated uint64 bucket_counts = 2; + } + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 10; + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 11; + + // The minimum value over (start_time, end_time]. + optional double min = 12; + + // The maximum value over (start_time, end_time]. + optional double max = 13; + + // ZeroThreshold may be optionally set to convey the width of the zero + // region. Where the zero region is defined as the closed interval + // [-ZeroThreshold, ZeroThreshold]. + // When ZeroThreshold is 0, zero count bucket stores values that cannot be + // expressed using the standard exponential formula as well as values that + // have been rounded to zero. + double zero_threshold = 14; +} + +// SummaryDataPoint is a single data point in a timeseries that describes the +// time-varying values of a Summary metric. The count and sum fields represent +// cumulative values. +message SummaryDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list may be empty (may contain 0 elements). + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // + // The attribute values SHOULD NOT contain empty values. + // The attribute values SHOULD NOT contain bytes values. + // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values, + // double values. + // The attribute values SHOULD NOT contain kvlist values. + // The behavior of software that receives attributes containing such values can be unpredictable. + // These restrictions can change in a minor release. + // The restrictions take origin from the OpenTelemetry specification: + // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // count is the number of values in the population. Must be non-negative. + fixed64 count = 4; + + // sum of the values in the population. If count is zero then this field + // must be zero. + // + // Note: Sum should only be filled out when measuring non-negative discrete + // events, and is assumed to be monotonic over the values of these events. + // Negative events *can* be recorded, but sum should not be filled out when + // doing so. This is specifically to enforce compatibility w/ OpenMetrics, + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#summary + double sum = 5; + + // Represents the value at a given quantile of a distribution. + // + // To record Min and Max values following conventions are used: + // - The 1.0 quantile is equivalent to the maximum value observed. + // - The 0.0 quantile is equivalent to the minimum value observed. + // + // See the following issue for more context: + // https://github.com/open-telemetry/opentelemetry-proto/issues/125 + message ValueAtQuantile { + // The quantile of a distribution. Must be in the interval + // [0.0, 1.0]. + double quantile = 1; + + // The value at the given quantile of a distribution. + // + // Quantile values must NOT be negative. + double value = 2; + } + + // (Optional) list of values at different quantiles of the distribution calculated + // from the current snapshot. The quantiles must be strictly increasing. + repeated ValueAtQuantile quantile_values = 6; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 8; +} + +// A representation of an exemplar, which is a sample input measurement. +// Exemplars also hold information about the environment when the measurement +// was recorded, for example the span and trace ID of the active span when the +// exemplar was recorded. +message Exemplar { + reserved 1; + + // The set of key/value pairs that were filtered out by the aggregator, but + // recorded alongside the original measurement. Only key/value pairs that were + // filtered out by the aggregator should be included + repeated opentelemetry.proto.common.v1.KeyValue filtered_attributes = 7; + + // time_unix_nano is the exact time when this exemplar was recorded + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 2; + + // The value of the measurement that was recorded. An exemplar is + // considered invalid when one of the recognized value fields is not present + // inside this oneof. + oneof value { + double as_double = 3; + sfixed64 as_int = 6; + } + + // (Optional) Span ID of the exemplar trace. + // span_id may be missing if the measurement is not recorded inside a trace + // or if the trace is not sampled. + bytes span_id = 4; + + // (Optional) Trace ID of the exemplar trace. + // trace_id may be missing if the measurement is not recorded inside a trace + // or if the trace is not sampled. + bytes trace_id = 5; +} + +// clang-format on +#endif + +namespace zen::otel { + +// Metrics + +enum class MetricsData : protozero::pbf_tag_type +{ + repeated_ResourceMetrics_resource_metrics = 1 +}; + +enum class ResourceMetrics : protozero::pbf_tag_type +{ + Resource_resource = 1, + repeated_ScopeMetrics_scope_metrics = 2, + string_schema_url = 3 +}; + +// A collection of Metrics produced by an Scope. +enum class ScopeMetrics : protozero::pbf_tag_type +{ + InstrumentationScope_scope = 1, + repeated_Metric_metrics = 2, + string_schema_url = 3 +}; + +enum class Metric : protozero::pbf_tag_type +{ + string_name = 1, + string_description = 2, + string_unit = 3, + + oneof_data_Gauge_gauge = 5, + oneof_data_Sum_sum = 7, + oneof_data_Histogram_histogram = 9, + oneof_data_ExponentialHistogram_exponential_histogram = 10, + oneof_data_Summary_summary = 11, + + repeated_KeyValue_metadata = 12 +}; + +enum class Gauge : protozero::pbf_tag_type +{ + repeated_NumberDataPoint_data_points = 1 +}; + +enum class Sum : protozero::pbf_tag_type +{ + repeated_NumberDataPoint_data_points = 1, + AggregationTemporality_aggregation_temporality = 2, + bool_is_monotonic = 3 +}; + +enum class Histogram : protozero::pbf_tag_type +{ + repeated_HistogramDataPoint_data_points = 1, + AggregationTemporality_aggregation_temporality = 2 +}; + +enum class ExponentialHistogram : protozero::pbf_tag_type +{ + repeated_ExponentialHistogramDataPoint_data_points = 1, + AggregationTemporality_aggregation_temporality = 2 +}; + +enum class Summary : protozero::pbf_tag_type +{ + repeated_SummaryDataPoint_data_points = 1 +}; + +enum AggregationTemporality +{ + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0, + AGGREGATION_TEMPORALITY_DELTA = 1, + AGGREGATION_TEMPORALITY_CUMULATIVE = 2 +}; + +enum DataPointFlags +{ + DATA_POINT_FLAGS_DO_NOT_USE = 0, + DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK = 1, +}; + +enum class NumberDataPoint : protozero::pbf_tag_type +{ + repeated_KeyValue_attributes = 7, + fixed64_start_time_unix_nano = 2, + fixed64_time_unix_nano = 3, + + oneof_value_double_as_double = 4, + oneof_value_sfixed64_as_int = 6, + + repeated_Exemplar_exemplars = 5, + uint32_flags = 8 +}; + +enum class HistogramDataPoint : protozero::pbf_tag_type +{ + repeated_KeyValue_attributes = 9, + fixed64_start_time_unix_nano = 2, + fixed64_time_unix_nano = 3, + fixed64_count = 4, + optional_double_sum = 5, + repeated_fixed64_bucket_counts = 6, + repeated_double_explicit_bounds = 7, + repeated_Exemplar_exemplars = 8, + uint32_flags = 10, + optional_double_min = 11, + optional_double_max = 12 +}; + +enum class Buckets : protozero::pbf_tag_type +{ + sint32_offset = 1, + repeated_uint64_bucket_counts = 2 +}; + +enum class ExponentialHistogramDataPoint : protozero::pbf_tag_type +{ + repeated_KeyValue_attributes = 1, + fixed64_start_time_unix_nano = 2, + fixed64_time_unix_nano = 3, + fixed64_count = 4, + optional_double_sum = 5, + sint32_scale = 6, + fixed64_zero_count = 7, + Buckets_positive = 8, + Buckets_negative = 9, + uint32_flags = 10, + repeated_Exemplar_exemplars = 11, + optional_double_min = 12, + optional_double_max = 13, + double_zero_threshold = 14 +}; + +enum class ValueAtQuantile : protozero::pbf_tag_type +{ + double_quantile = 1, + double_value = 2 +}; + +enum class SummaryDataPoint : protozero::pbf_tag_type +{ + repeated_KeyValue_attributes = 7, + fixed64_start_time_unix_nano = 2, + fixed64_time_unix_nano = 3, + fixed64_count = 4, + double_sum = 5, + repeated_ValueAtQuantile_quantile_values = 6, + uint32_flags = 8 +}; + +enum class Exemplar : protozero::pbf_tag_type +{ + repeated_KeyValue_filtered_attributes = 7, + fixed64_time_unix_nano = 2, + oneof_value_double_as_double = 3, + oneof_value_sfixed64_as_int = 6, + bytes_span_id = 4, + bytes_trace_id = 5 +}; + +} // namespace zen::otel
\ No newline at end of file diff --git a/src/zentelemetry/otelprotozero.h b/src/zentelemetry/otelprotozero.h new file mode 100644 index 000000000..8ad69e766 --- /dev/null +++ b/src/zentelemetry/otelprotozero.h @@ -0,0 +1,166 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <protozero/types.hpp> + +////////////////////////////////////////////////////////////////////////// +// +// OTEL .proto definitions, for reference +// + +#if 0 + +// clang-format off + +//////////////////////////////////////////////////////////////////////// +// resource/v1/resource.proto +// + +// Resource information. +message Resource { + // Set of attributes that describe the resource. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // + // The attribute values SHOULD NOT contain empty values. + // The attribute values SHOULD NOT contain bytes values. + // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values, + // double values. + // The attribute values SHOULD NOT contain kvlist values. + // The behavior of software that receives attributes containing such values can be unpredictable. + // These restrictions can change in a minor release. + // The restrictions take origin from the OpenTelemetry specification: + // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // The number of dropped attributes. If the value is 0, then + // no attributes were dropped. + uint32 dropped_attributes_count = 2; + + // Set of entities that participate in this Resource. + // + // Note: keys in the references MUST exist in attributes of this message. + // + // Status: [Development] + repeated opentelemetry.proto.common.v1.EntityRef entity_refs = 3; +} + +///////////////////////////////////////////////////////////////////////// +// common/v1/commmon.proto +// + +// Represents any type of attribute value. AnyValue may contain a +// primitive value such as a string or integer or it may contain an arbitrary nested +// object containing arrays, key-value lists and primitives. +message AnyValue { + // The value is one of the listed fields. It is valid for all values to be unspecified + // in which case this AnyValue is considered to be "empty". + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + ArrayValue array_value = 5; + KeyValueList kvlist_value = 6; + bytes bytes_value = 7; + } +} + +// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message +// since oneof in AnyValue does not allow repeated fields. +message ArrayValue { + // Array of values. The array may be empty (contain 0 elements). + repeated AnyValue values = 1; +} + +// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message +// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need +// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to +// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches +// are semantically equivalent. +message KeyValueList { + // A collection of key/value pairs of key-value pairs. The list may be empty (may + // contain 0 elements). + // The keys MUST be unique (it is not allowed to have more than one + // value with the same key). + repeated KeyValue values = 1; +} + +// Represents a key-value pair that is used to store Span attributes, Link +// attributes, etc. +message KeyValue { + // The key name of the pair. + string key = 1; + + // The value of the pair. + AnyValue value = 2; +} + +// InstrumentationScope is a message representing the instrumentation scope information +// such as the fully qualified name and version. +message InstrumentationScope { + // A name denoting the Instrumentation scope. + // An empty instrumentation scope name means the name is unknown. + string name = 1; + + // Defines the version of the instrumentation scope. + // An empty instrumentation scope version means the version is unknown. + string version = 2; + + // Additional attributes that describe the scope. [Optional]. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated KeyValue attributes = 3; + + // The number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 4; +} + +// clang-format on +#endif + +////////////////////////////////////////////////////////////////////////// + +namespace zen::otel { + +enum class KeyValueList : protozero::pbf_tag_type +{ + repeated_KeyValue_values = 1 +}; + +enum class KeyValue : protozero::pbf_tag_type +{ + string_key = 1, + AnyValue_value = 2 +}; + +enum class Resource : protozero::pbf_tag_type +{ + repeated_KeyValue_attributes = 1, + uint32_dropped_attributes_count = 2, + repeated_EntityRef_entity_refs = 3 +}; + +enum class AnyValue : protozero::pbf_tag_type +{ + string_string_value = 1, + bool_bool_value = 2, + int64_int_value = 3, + double_double_value = 4, + ArrayValue_array_value = 5, + KeyValueList_kvlist_value = 6, + bytes_bytes_value = 7 +}; + +enum class InstrumentationScope : protozero::pbf_tag_type +{ + string_name = 1, + string_version = 2, + repeated_KeyValue_attributes = 3, + uint32_dropped_attributes_count = 4 +}; + +} // namespace zen::otel diff --git a/src/zentelemetry/oteltraceprotozero.h b/src/zentelemetry/oteltraceprotozero.h new file mode 100644 index 000000000..8b80c8f7f --- /dev/null +++ b/src/zentelemetry/oteltraceprotozero.h @@ -0,0 +1,474 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "otelprotozero.h" + +////////////////////////////////////////////////////////////////////////// +// +// OTEL .proto definitions, for reference +// + +#if 0 + +// clang-format off + +///////////////////////////////////////////////////////////////////////// +// trace/v1/trace.proto +// + +// TracesData represents the traces data that can be stored in a persistent storage, +// OR can be embedded by other protocols that transfer OTLP traces data but do +// not implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message TracesData { + // An array of ResourceSpans. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceSpans resource_spans = 1; +} + +// A collection of ScopeSpans from a Resource. +message ResourceSpans { + reserved 1000; + + // The resource for the spans in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeSpans that originate from a resource. + repeated ScopeSpans scope_spans = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_spans" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Spans produced by an InstrumentationScope. +message ScopeSpans { + // The instrumentation scope information for the spans in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Spans that originate from an instrumentation scope. + repeated Span spans = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the span data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all spans and span events in the "spans" field. + string schema_url = 3; +} + +// A Span represents a single operation performed by a single component of the system. +// +// The next available field id is 17. +message Span { + // A unique identifier for a trace. All spans from the same trace share + // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is required. + bytes trace_id = 1; + + // A unique identifier for a span within a trace, assigned when the span + // is created. The ID is an 8-byte array. An ID with all zeroes OR of length + // other than 8 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is required. + bytes span_id = 2; + + // trace_state conveys information about request position in multiple distributed tracing graphs. + // It is a trace_state in w3c-trace-context format: https://www.w3.org/TR/trace-context/#tracestate-header + // See also https://github.com/w3c/distributed-tracing for more details about this field. + string trace_state = 3; + + // The `span_id` of this span's parent span. If this is a root span, then this + // field must be empty. The ID is an 8-byte array. + bytes parent_span_id = 4; + + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether a span's parent + // is remote. The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // When creating span messages, if the message is logically forwarded from another source + // with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD + // be copied as-is. If creating from a source that does not have an equivalent flags field + // (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST + // be set to zero. + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // + // [Optional]. + fixed32 flags = 16; + + // A description of the span's operation. + // + // For example, the name can be a qualified method name or a file name + // and a line number where the operation is called. A best practice is to use + // the same display name at the same call point in an application. + // This makes it easier to correlate spans in different traces. + // + // This field is semantically required to be set to non-empty string. + // Empty value is equivalent to an unknown span name. + // + // This field is required. + string name = 5; + + // SpanKind is the type of span. Can be used to specify additional relationships between spans + // in addition to a parent/child relationship. + enum SpanKind { + // Unspecified. Do NOT use as default. + // Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. + SPAN_KIND_UNSPECIFIED = 0; + + // Indicates that the span represents an internal operation within an application, + // as opposed to an operation happening at the boundaries. Default value. + SPAN_KIND_INTERNAL = 1; + + // Indicates that the span covers server-side handling of an RPC or other + // remote network request. + SPAN_KIND_SERVER = 2; + + // Indicates that the span describes a request to some remote service. + SPAN_KIND_CLIENT = 3; + + // Indicates that the span describes a producer sending a message to a broker. + // Unlike CLIENT and SERVER, there is often no direct critical path latency relationship + // between producer and consumer spans. A PRODUCER span ends when the message was accepted + // by the broker while the logical processing of the message might span a much longer time. + SPAN_KIND_PRODUCER = 4; + + // Indicates that the span describes consumer receiving a message from a broker. + // Like the PRODUCER kind, there is often no direct critical path latency relationship + // between producer and consumer spans. + SPAN_KIND_CONSUMER = 5; + } + + // Distinguishes between spans generated in a particular context. For example, + // two spans with the same name may be distinguished using `CLIENT` (caller) + // and `SERVER` (callee) to identify queueing latency associated with the span. + SpanKind kind = 6; + + // The start time of the span. On the client side, this is the time + // kept by the local machine where the span execution starts. On the server side, this + // is the time when the server's application handler starts running. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 start_time_unix_nano = 7; + + // The end time of the span. On the client side, this is the time + // kept by the local machine where the span execution ends. On the server side, this + // is the time when the server application handler stops running. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 end_time_unix_nano = 8; + + // A collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. Examples of attributes: + // + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "example.com/myattribute": true + // "example.com/score": 10.239 + // + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // + // The attribute values SHOULD NOT contain empty values. + // The attribute values SHOULD NOT contain bytes values. + // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values, + // double values. + // The attribute values SHOULD NOT contain kvlist values. + // The behavior of software that receives attributes containing such values can be unpredictable. + // These restrictions can change in a minor release. + // The restrictions take origin from the OpenTelemetry specification: + // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; + + // The number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 10; + + // Event is a time-stamped annotation of the span, consisting of user-supplied + // text description and key-value pairs. + message Event { + // The time the event occurred. + fixed64 time_unix_nano = 1; + + // The name of the event. + // This field is semantically required to be set to non-empty string. + string name = 2; + + // A collection of attribute key/value pairs on the event. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // + // The attribute values SHOULD NOT contain empty values. + // The attribute values SHOULD NOT contain bytes values. + // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values, + // double values. + // The attribute values SHOULD NOT contain kvlist values. + // The behavior of software that receives attributes containing such values can be unpredictable. + // These restrictions can change in a minor release. + // The restrictions take origin from the OpenTelemetry specification: + // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; + + // The number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 4; + } + + // A collection of Event items. + repeated Event events = 11; + + // The number of dropped events. If the value is 0, then no + // events were dropped. + uint32 dropped_events_count = 12; + + // A pointer from the current span to another span in the same trace or in a + // different trace. For example, this can be used in batching operations, + // where a single batch handler processes multiple requests from different + // traces or when the handler receives a request from a different project. + message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; + + // The trace_state associated with the link. + string trace_state = 3; + + // A collection of attribute key/value pairs on the link. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // + // The attribute values SHOULD NOT contain empty values. + // The attribute values SHOULD NOT contain bytes values. + // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values, + // double values. + // The attribute values SHOULD NOT contain kvlist values. + // The behavior of software that receives attributes containing such values can be unpredictable. + // These restrictions can change in a minor release. + // The restrictions take origin from the OpenTelemetry specification: + // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; + + // The number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 5; + + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether the link is remote. + // The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. + // + // [Optional]. + fixed32 flags = 6; + } + + // A collection of Links, which are references from this span to a span + // in the same or different trace. + repeated Link links = 13; + + // The number of dropped links after the maximum size was + // enforced. If this value is 0, then no links were dropped. + uint32 dropped_links_count = 14; + + // An optional final status for this span. Semantically when Status isn't set, it means + // span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). + Status status = 15; +} + +// The Status type defines a logical error model that is suitable for different +// programming environments, including REST APIs and RPC APIs. +message Status { + reserved 1; + + // A developer-facing human readable error message. + string message = 2; + + // For the semantics of status codes see + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status + enum StatusCode { + // The default status. + STATUS_CODE_UNSET = 0; + // The Span has been validated by an Application developer or Operator to + // have completed successfully. + STATUS_CODE_OK = 1; + // The Span contains an error. + STATUS_CODE_ERROR = 2; + }; + + // The status code. + StatusCode code = 3; +} + +// SpanFlags represents constants used to interpret the +// Span.flags field, which is protobuf 'fixed32' type and is to +// be used as bit-fields. Each non-zero value defined in this enum is +// a bit-mask. To extract the bit-field, for example, use an +// expression like: +// +// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) +// +// See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. +// +// Note that Span flags were introduced in version 1.1 of the +// OpenTelemetry protocol. Older Span producers do not set this +// field, consequently consumers should not rely on the absence of a +// particular flag bit to indicate the presence of a particular feature. +enum SpanFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + SPAN_FLAGS_DO_NOT_USE = 0; + + // Bits 0-7 are used for trace flags. + SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; + + // Bits 8 and 9 are used to indicate that the parent span or link span is remote. + // Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. + // Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. + SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100; + SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200; + + // Bits 10-31 are reserved for future use. +} + +// clang-format on +#endif + +namespace zen::otel::pbf { + +// Traces + +enum class TracesData : protozero::pbf_tag_type +{ + repeated_ResourceSpans_resource_spans = 1 +}; + +enum class ResourceSpans : protozero::pbf_tag_type +{ + Resource_resource = 1, + repeated_ScopeSpans_scope_spans = 2, + string_schema_url = 3 +}; + +enum class ScopeSpans : protozero::pbf_tag_type +{ + InstrumentationScope_scope = 1, + repeated_Span_spans = 2, + string_schema_url = 3 +}; + +enum SpanFlags +{ + SPAN_FLAGS_DO_NOT_USE = 0, + SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF, + SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100, + SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200, +}; + +enum SpanKind +{ + SPAN_KIND_UNSPECIFIED = 0, + SPAN_KIND_INTERNAL = 1, + SPAN_KIND_SERVER = 2, + SPAN_KIND_CLIENT = 3, + SPAN_KIND_PRODUCER = 4, + SPAN_KIND_CONSUMER = 5 +}; + +enum class Span_Event : protozero::pbf_tag_type +{ + fixed64_time_unix_nano = 1, + string_name = 2, + repeated_KeyValue_attributes = 3, + uint32_dropped_attributes_count = 4 +}; + +enum class Span_Link : protozero::pbf_tag_type +{ + bytes_trace_id = 1, + bytes_span_id = 2, + string_trace_state = 3, + repeated_KeyValue_attributes = 4, + uint32_dropped_attributes_count = 5, + fixed32_flags = 6 +}; + +enum class Span : protozero::pbf_tag_type +{ + required_bytes_trace_id = 1, + required_bytes_span_id = 2, + string_trace_state = 3, + bytes_parent_span_id = 4, + fixed32_flags = 16, + required_string_name = 5, + SpanKind_kind = 6, + required_fixed64_start_time_unix_nano = 7, + required_fixed64_end_time_unix_nano = 8, + repeated_KeyValue_attributes = 9, + uint32_dropped_attributes_count = 10, + repeated_Event_events = 11, + uint32_dropped_events_count = 12, + repeated_Link_links = 13, + uint32_dropped_links_count = 14, + Status_status = 15 +}; + +enum StatusCode +{ + STATUS_CODE_UNSET = 0, + STATUS_CODE_OK = 1, + STATUS_CODE_ERROR = 2 +}; + +enum class Status : protozero::pbf_tag_type +{ + string_message = 2, + StatusCode_code = 3 +}; + +} // namespace zen::otel::pbf diff --git a/src/zentelemetry/otlpencoder.cpp b/src/zentelemetry/otlpencoder.cpp new file mode 100644 index 000000000..677545066 --- /dev/null +++ b/src/zentelemetry/otlpencoder.cpp @@ -0,0 +1,478 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zentelemetry/otlpencoder.h" + +#include <zenbase/zenbase.h> +#include <zentelemetry/otlptrace.h> + +#include <spdlog/sinks/sink.h> +#include <zencore/testing.h> + +#include <protozero/buffer_string.hpp> +#include <protozero/pbf_builder.hpp> + +#include "otellogprotozero.h" +#include "otelmetricsprotozero.h" +#include "otelprotozero.h" +#include "oteltraceprotozero.h" + +#if ZEN_WITH_OTEL + +namespace zen { + +OtlpEncoder::OtlpEncoder() +{ +} + +OtlpEncoder::~OtlpEncoder() +{ +} + +static int +MapSeverity(const spdlog::level::level_enum Level) +{ + switch (Level) + { + case spdlog::level::critical: + return otel::SEVERITY_NUMBER_FATAL; + case spdlog::level::err: + return otel::SEVERITY_NUMBER_ERROR; + case spdlog::level::warn: + return otel::SEVERITY_NUMBER_WARN; + case spdlog::level::info: + return otel::SEVERITY_NUMBER_INFO; + case spdlog::level::debug: + return otel::SEVERITY_NUMBER_DEBUG; + default: + case spdlog::level::trace: + return otel::SEVERITY_NUMBER_TRACE; + } +} + +static const char* +MapSeverityText(const spdlog::level::level_enum Level) +{ + switch (Level) + { + case spdlog::level::critical: + return "fatal"; + case spdlog::level::err: + return "error"; + case spdlog::level::warn: + return "warn"; + case spdlog::level::info: + return "info"; + case spdlog::level::debug: + return "debug"; + default: + case spdlog::level::trace: + return "trace"; + } +} + +std::string +OtlpEncoder::FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const +{ + std::string Data; + + // LogsData + { + protozero::pbf_builder<otel::LogsData> Builder{Data}; + + // ResourceLogs + { + protozero::pbf_builder<otel::ResourceLogs> RlBuilder{Builder, otel::LogsData::required_repeated_ResourceLogs_resource_logs}; + + // ResourceLogs / Resource + { + protozero::pbf_builder<otel::Resource> Res{RlBuilder, otel::ResourceLogs::optional_Resource_resource}; + + AppendResourceAttributes(Res); + } + + // ScopeLogs scope_logs + { + protozero::pbf_builder<otel::ScopeLogs> SlBuilder{RlBuilder, otel::ResourceLogs::required_repeated_ScopeLogs_scope_logs}; + + { + protozero::pbf_builder<otel::InstrumentationScope> IsBuilder{SlBuilder, + otel::ScopeLogs::required_InstrumentationScope_scope}; + + IsBuilder.add_string(otel::InstrumentationScope::string_name, Msg.logger_name.data(), Msg.logger_name.size()); + } + + // LogRecord log_records + { + protozero::pbf_builder<otel::LogRecord> LrBuilder{SlBuilder, otel::ScopeLogs::required_repeated_LogRecord_log_records}; + + LrBuilder.add_fixed64(otel::LogRecord::required_fixed64_time_unix_nano, + std::chrono::duration_cast<std::chrono::nanoseconds>(Msg.time.time_since_epoch()).count()); + + const int Severity = MapSeverity(Msg.level); + + LrBuilder.add_enum(otel::LogRecord::optional_SeverityNumber_severity_number, Severity); + + LrBuilder.add_string(otel::LogRecord::optional_string_severity_text, MapSeverityText(Msg.level)); + + otel::TraceId TraceId; + const otel::SpanId SpanId = otel::Span::GetCurrentSpanId(TraceId); + + if (SpanId && TraceId) + { + LrBuilder.add_bytes(otel::LogRecord::optional_bytes_trace_id, TraceId.GetData(), TraceId.kSize); + LrBuilder.add_bytes(otel::LogRecord::optional_bytes_span_id, SpanId.GetData(), SpanId.kSize); + } + + // body + { + protozero::pbf_builder<otel::AnyValue> BodyBuilder{LrBuilder, otel::LogRecord::optional_anyvalue_body}; + + BodyBuilder.add_string(otel::AnyValue::string_string_value, Msg.payload.data(), Msg.payload.size()); + } + + // attributes + + { + protozero::pbf_builder<otel::KeyValue> KvBuilder{LrBuilder, otel::LogRecord::optional_repeated_kv_attributes}; + KvBuilder.add_string(otel::KeyValue::string_key, "thread_id"); + + { + protozero::pbf_builder<otel::AnyValue> AvBuilder{KvBuilder, otel::KeyValue::AnyValue_value}; + + AvBuilder.add_int64(otel::AnyValue::int64_int_value, Msg.thread_id); + } + } + } + } + } + } + + return Data; +} + +std::string +OtlpEncoder::FormatOtelMetrics() const +{ + std::string Data; + +# if 0 + static int64_t LastNanos = 0; + + const int64_t NowNanos = + std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count(); + + // MetricsData + { + protozero::pbf_builder<otel::MetricsData> Builder{Data}; + + // ResourceMetrics + protozero::pbf_builder<otel::ResourceMetrics> Rm{Builder, otel::MetricsData::repeated_ResourceMetrics_resource_metrics}; + + { + protozero::pbf_builder<otel::Resource> Res{Rm, otel::ResourceMetrics::Resource_resource}; + + AppendResourceAttributes(Res); + } + + // ScopeMetrics + protozero::pbf_builder<otel::ScopeMetrics> Sm{Rm, otel::ResourceMetrics::repeated_ScopeMetrics_scope_metrics}; + + { + // InstrumentationScope + protozero::pbf_builder<otel::InstrumentationScope> Is{Sm, otel::ScopeMetrics::InstrumentationScope_scope}; + Is.add_string(otel::InstrumentationScope::string_name, "scope_name"); + } + + { + protozero::pbf_builder<otel::Metric> Metric{Sm, otel::ScopeMetrics::repeated_Metric_metrics}; + Metric.add_string(otel::Metric::string_name, "metric_name"); + Metric.add_string(otel::Metric::string_unit, "KiB"); + + // Gauge + { + protozero::pbf_builder<otel::Gauge> Gauge{Metric, otel::Metric::oneof_data_Gauge_gauge}; + + protozero::pbf_builder<otel::NumberDataPoint> Dp{Gauge, otel::Gauge::repeated_NumberDataPoint_data_points}; + Dp.add_fixed64(otel::NumberDataPoint::fixed64_time_unix_nano, NowNanos); + Dp.add_fixed64(otel::NumberDataPoint::fixed64_start_time_unix_nano, LastNanos); + Dp.add_sfixed64(otel::NumberDataPoint::oneof_value_sfixed64_as_int, rand() * 470 / RAND_MAX); + } + } + + const int RequestCount = rand() % 600; + + { + protozero::pbf_builder<otel::Metric> Metric{Sm, otel::ScopeMetrics::repeated_Metric_metrics}; + Metric.add_string(otel::Metric::string_name, "request_count"); + Metric.add_string(otel::Metric::string_unit, "requests"); + + static int SumValue = 0; + SumValue += RequestCount; + + // Sum + { + protozero::pbf_builder<otel::Sum> Sum{Metric, otel::Metric::oneof_data_Sum_sum}; + Sum.add_enum(otel::Sum::AggregationTemporality_aggregation_temporality, otel::AGGREGATION_TEMPORALITY_CUMULATIVE); + Sum.add_bool(otel::Sum::bool_is_monotonic, true); + + protozero::pbf_builder<otel::NumberDataPoint> Dp{Sum, otel::Sum::repeated_NumberDataPoint_data_points}; + Dp.add_fixed64(otel::NumberDataPoint::fixed64_time_unix_nano, NowNanos); + Dp.add_fixed64(otel::NumberDataPoint::fixed64_start_time_unix_nano, LastNanos); + Dp.add_double(otel::NumberDataPoint::oneof_value_double_as_double, SumValue); + } + } + + { + protozero::pbf_builder<otel::Metric> Metric{Sm, otel::ScopeMetrics::repeated_Metric_metrics}; + Metric.add_string(otel::Metric::string_name, "request_latency"); + Metric.add_string(otel::Metric::string_unit, "ms"); + + // Histogram + { + protozero::pbf_builder<otel::Histogram> Histogram{Metric, otel::Metric::oneof_data_Histogram_histogram}; + Histogram.add_enum(otel::Histogram::AggregationTemporality_aggregation_temporality, + otel::AGGREGATION_TEMPORALITY_CUMULATIVE); + + protozero::pbf_builder<otel::HistogramDataPoint> Dp{Histogram, otel::Histogram::repeated_HistogramDataPoint_data_points}; + Dp.add_fixed64(otel::HistogramDataPoint::fixed64_time_unix_nano, NowNanos); + Dp.add_fixed64(otel::HistogramDataPoint::fixed64_start_time_unix_nano, LastNanos); + + // Simulated latency value + + double Sum = 0, Min = 0, Max = 0; + int Buckets[6] = {0}; + + for (int i = 0; i < RequestCount; ++i) + { + const int Latency = rand() % 250; + + if (i == 0 || Latency < Min) + { + Min = Latency; + } + if (i == 0 || Latency > Max) + { + Max = Latency; + } + + Sum += Latency; + + if (Latency >= 0 && Latency < 10) + { + ++Buckets[0]; // [0,10) + } + else if (Latency >= 10 && Latency < 25) + { + ++Buckets[1]; // [10,25) + } + else if (Latency >= 25 && Latency < 50) + { + ++Buckets[2]; // [25,50) + } + else if (Latency >= 50 && Latency < 100) + { + ++Buckets[3]; // [50,100) + } + else if (Latency >= 100 && Latency < 250) + { + ++Buckets[4]; // [100,250) + } + else + { + ++Buckets[5]; // [250,+inf) + } + } + + Dp.add_fixed64(otel::HistogramDataPoint::fixed64_count, RequestCount); + Dp.add_double(otel::HistogramDataPoint::optional_double_sum, Sum); + Dp.add_double(otel::HistogramDataPoint::optional_double_min, Min); + Dp.add_double(otel::HistogramDataPoint::optional_double_max, Max); + + // Bucket bounds + Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 10.0); + Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 25.0); + Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 50.0); + Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 100.0); + Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 250.0); + + // Buckets + for (int i = 0; i < 6; ++i) + { + Dp.add_fixed64(otel::HistogramDataPoint::repeated_fixed64_bucket_counts, Buckets[i]); + } + } + } + +# if 0 + { + protozero::pbf_builder<otel::Metric> Metric{Sm, otel::ScopeMetrics::repeated_Metric_metrics}; + Metric.add_string(otel::Metric::string_name, "request_payload"); + Metric.add_string(otel::Metric::string_unit, "KiB"); + + // ExponentialHistogram + { + protozero::pbf_builder<otel::ExponentialHistogram> ExponentialHistogram{ + Metric, otel::Metric::oneof_data_ExponentialHistogram_exponential_histogram}; + ExponentialHistogram.add_enum(otel::ExponentialHistogram::AggregationTemporality_aggregation_temporality, + otel::AGGREGATION_TEMPORALITY_CUMULATIVE); + + protozero::pbf_builder<otel::ExponentialHistogramDataPoint> Dp{ + ExponentialHistogram, otel::ExponentialHistogram::repeated_ExponentialHistogramDataPoint_data_points}; + Dp.add_fixed64(otel::ExponentialHistogramDataPoint::fixed64_time_unix_nano, NowNanos); + Dp.add_fixed64(otel::ExponentialHistogramDataPoint::fixed64_start_time_unix_nano, LastNanos); + + // Simulated payload size values + + int64_t sum = 0; + int count = 0; + + for (int i = 0; i < RequestCount; ++i) + { + const int PayloadSize = rand() % 1'500'000; // [0,1500000) bytes + + sum += PayloadSize; + ++count; + } + + Dp.add_fixed64(otel::ExponentialHistogramDataPoint::fixed64_count, count); + Dp.add_double(otel::ExponentialHistogramDataPoint::optional_double_sum, sum); + Dp.add_sint32(otel::ExponentialHistogramDataPoint::sint32_scale, 4); + } + } +# endif + } + LastNanos = NowNanos; +# endif + return Data; +} + +void +OtlpEncoder::AddResourceAttribute(const std::string_view& Key, const std::string_view& Value) +{ + m_ResourceLock.WithExclusiveLock([&] { m_ResourceAttributes[std::string(Key)] = std::string(Value); }); +} + +void +OtlpEncoder::AddResourceAttribute(const std::string_view& Key, int64_t Value) +{ + m_ResourceLock.WithExclusiveLock([&] { m_ResourceIntAttributes[std::string(Key)] = Value; }); +} + +void +OtlpEncoder::AppendResourceAttributes(protozero::pbf_builder<otel::Resource>& Res) const +{ + using namespace otel; + + m_ResourceLock.WithSharedLock([&] { + for (auto const& [K, V] : m_ResourceAttributes) + { + protozero::pbf_builder<otel::KeyValue> KvBuilder{Res, otel::Resource::repeated_KeyValue_attributes}; + KvBuilder.add_string(otel::KeyValue::string_key, K.c_str()); + protozero::pbf_builder<otel::AnyValue> AnyBuilder{KvBuilder, otel::KeyValue::AnyValue_value}; + AnyBuilder.add_string(otel::AnyValue::string_string_value, V.c_str()); + } + + for (auto const& [K, V] : m_ResourceIntAttributes) + { + protozero::pbf_builder<otel::KeyValue> KvBuilder{Res, otel::Resource::repeated_KeyValue_attributes}; + KvBuilder.add_string(otel::KeyValue::string_key, K.c_str()); + protozero::pbf_builder<otel::AnyValue> AnyBuilder{KvBuilder, otel::KeyValue::AnyValue_value}; + AnyBuilder.add_int64(otel::AnyValue::int64_int_value, V); + } + }); +} + +template<typename ParentBuilder> +static void +AppendAttributesToBuilder(ParentBuilder& Parent, auto KeyValueField, const otel::AttributePair* Attrs) +{ + for (const otel::AttributePair* Attr = Attrs; Attr != nullptr; Attr = Attr->Next) + { + protozero::pbf_builder<otel::KeyValue> KvBuilder{Parent, KeyValueField}; + KvBuilder.add_string(otel::KeyValue::string_key, Attr->Key); + + protozero::pbf_builder<otel::AnyValue> AnyBuilder{KvBuilder, otel::KeyValue::AnyValue_value}; + + if (Attr->IsNumeric()) + { + AnyBuilder.add_int64(otel::AnyValue::int64_int_value, Attr->GetNumericValue()); + } + else if (Attr->IsString()) + { + AnyBuilder.add_string(otel::AnyValue::string_string_value, Attr->GetStringValue()); + } + } +} + +std::string +OtlpEncoder::FormatOtelTrace(zen::otel::TraceId Id, std::span<const zen::otel::Span*> Spans) const +{ + std::string Data; + + using namespace otel; + + // TracesData + { + protozero::pbf_builder<pbf::TracesData> Builder{Data}; + + { + protozero::pbf_builder<pbf::ResourceSpans> Rs{Builder, pbf::TracesData::repeated_ResourceSpans_resource_spans}; + + { + protozero::pbf_builder<otel::Resource> Res{Rs, pbf::ResourceSpans::Resource_resource}; + + AppendResourceAttributes(Res); + } + + for (const zen::otel::Span* SpanPtr : Spans) + { + protozero::pbf_builder<pbf::ScopeSpans> Ss{Rs, pbf::ResourceSpans::repeated_ScopeSpans_scope_spans}; + + { + // InstrumentationScope + protozero::pbf_builder<otel::InstrumentationScope> Is{Ss, pbf::ScopeSpans::InstrumentationScope_scope}; + Is.add_string(otel::InstrumentationScope::string_name, "scope_name"); + } + + const SpanId ThisSpanId = SpanPtr->GetSpanId(); + + { + protozero::pbf_builder<pbf::Span> Sb{Ss, pbf::ScopeSpans::repeated_Span_spans}; + + Sb.add_bytes(pbf::Span::required_bytes_trace_id, Id.GetData(), TraceId::kSize); + Sb.add_bytes(pbf::Span::required_bytes_span_id, ThisSpanId.GetData(), SpanId::kSize); + // Sb.add_string(pbf::Span::string_trace_state, "state-value"); + // + if (const otel::Span* ParentSpan = SpanPtr->GetParentSpan()) + { + const SpanId ParentSpanId = ParentSpan->GetSpanId(); + Sb.add_bytes(pbf::Span::bytes_parent_span_id, ParentSpanId.GetData(), SpanId::kSize); + } + + Sb.add_fixed32(pbf::Span::fixed32_flags, 0); + Sb.add_string(pbf::Span::required_string_name, SpanPtr->GetName()); + Sb.add_enum(pbf::Span::SpanKind_kind, (int)SpanPtr->GetKind()); + Sb.add_fixed64(pbf::Span::required_fixed64_start_time_unix_nano, SpanPtr->GetStartTime()); + Sb.add_fixed64(pbf::Span::required_fixed64_end_time_unix_nano, SpanPtr->GetEndTime()); + + AppendAttributesToBuilder(Sb, pbf::Span::repeated_KeyValue_attributes, SpanPtr->GetAttributes()); + + for (const otel::Event* Event = SpanPtr->GetEvents(); Event != nullptr; Event = Event->NextEvent) + { + protozero::pbf_builder<pbf::Span_Event> EventBuilder{Sb, pbf::Span::repeated_Event_events}; + EventBuilder.add_fixed64(pbf::Span_Event::fixed64_time_unix_nano, Event->Timestamp); + EventBuilder.add_string(pbf::Span_Event::string_name, Event->Name); + + AppendAttributesToBuilder(EventBuilder, pbf::Span_Event::repeated_KeyValue_attributes, Event->Attributes); + } + } + } + } + } + + return Data; +} + +} // namespace zen + +#endif diff --git a/src/zentelemetry/otlptrace.cpp b/src/zentelemetry/otlptrace.cpp new file mode 100644 index 000000000..b634eacf6 --- /dev/null +++ b/src/zentelemetry/otlptrace.cpp @@ -0,0 +1,398 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zentelemetry/otlptrace.h" +#include "oteltraceprotozero.h" + +#if ZEN_WITH_OTEL + +# include <zencore/endian.h> +# include <zencore/memory/memoryarena.h> +# include <zencore/session.h> +# include <zencore/testing.h> +# include <zencore/uid.h> + +# include <EASTL/list.h> +# include <EASTL/vector.h> + +# include <protozero/pbf_builder.hpp> +# include <random> + +namespace zen::otel { + +////////////////////////////////////////////////////////////////////////// + +uint64_t +NowInNanoseconds() +{ + const uint64_t NowNanos = + std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count(); + return static_cast<uint64_t>(NowNanos); +} + +////////////////////////////////////////////////////////////////////////// + +/** + * xorshiftr128+ random number generator + * + * https://en.wikipedia.org/wiki/Xorshift#xorshift+ + */ +class XorShiftr128p +{ +public: + XorShiftr128p() + { + std::seed_seq SeedSeq{1, 2, 3, 5}; + do + { + SeedSeq.generate(reinterpret_cast<uint32_t*>(m_State), reinterpret_cast<uint32_t*>(&m_State[2])); + } while (m_State[0] == 0 && m_State[1] == 0); + } + + uint64_t Next() + { + uint64_t x = m_State[0]; + uint64_t const y = m_State[1]; + m_State[0] = y; + x ^= x << 23; + x ^= x >> 17; + x ^= y; + m_State[1] = x + y; + return x; + } + +private: + uint64_t m_State[2]; +}; + +////////////////////////////////////////////////////////////////////////// + +struct TraceState +{ + Span* ActiveSpan = nullptr; + TraceId CurrentTraceId; +}; + +thread_local TraceState t_TraceState; + +////////////////////////////////////////////////////////////////////////// + +Span::Span(MemoryArena& Arena, std::string_view Name, Span* SpanChain) +: m_Arena(Arena) +, m_SpanChain(SpanChain) +, m_SpanId(SpanId::NewSpanId()) +, m_Name(m_Arena.DuplicateString(Name)) +{ + if (Span* ActiveSpan = t_TraceState.ActiveSpan) + { + m_ParentSpan = ActiveSpan; + m_NextSibling = ActiveSpan->m_FirstChild; + ActiveSpan->m_FirstChild = this; + } + else + { + m_ParentSpan = nullptr; + t_TraceState.CurrentTraceId = TraceId::NewTraceId(); + } + + t_TraceState.ActiveSpan = this; + m_StartTime = NowInNanoseconds(); +} + +Span::~Span() +{ + End(); +} + +void +Span::End() +{ + if (m_Ended) + { + return; + } + + ZEN_ASSERT(t_TraceState.ActiveSpan == this); + + m_Ended = true; + t_TraceState.ActiveSpan = m_ParentSpan; + m_EndTime = NowInNanoseconds(); +} + +Span* +Span::GetCurrentSpan() +{ + return t_TraceState.ActiveSpan; +} + +SpanId +Span::GetCurrentSpanId(TraceId& OutTraceId) +{ + if (t_TraceState.ActiveSpan) + { + OutTraceId = t_TraceState.CurrentTraceId; + return t_TraceState.ActiveSpan->m_SpanId; + } + else + { + OutTraceId = TraceId(); + return SpanId(); + } +} + +void +Span::AddEvent(std::string_view Name) +{ + Event* NewEvent = new (m_Arena) Event(); + NewEvent->Name = m_Arena.DuplicateString(Name); + NewEvent->Timestamp = NowInNanoseconds(); + NewEvent->NextEvent = m_Events; + m_Events = NewEvent; +} + +void +Span::AddEvent(std::string_view Name, uint64_t Timestamp) +{ + Event* NewEvent = new (m_Arena) Event(); + NewEvent->Name = m_Arena.DuplicateString(Name); + NewEvent->Timestamp = Timestamp; + NewEvent->NextEvent = m_Events; + m_Events = NewEvent; +} + +void +Span::AddEvent(std::string_view Name, const AttributeList& Attributes) +{ + Event* NewEvent = new (m_Arena) Event(); + NewEvent->Name = m_Arena.DuplicateString(Name); + NewEvent->Timestamp = NowInNanoseconds(); + NewEvent->NextEvent = m_Events; + m_Events = NewEvent; + + for (auto& [K, V] : Attributes) + { + AttributePair* NewAttr = new (m_Arena) AttributePair(); + NewAttr->Key = m_Arena.DuplicateString(K); + NewAttr->SetStringValue(m_Arena.DuplicateString(V)); + NewAttr->Next = NewEvent->Attributes; + NewEvent->Attributes = NewAttr; + } +} + +void +Span::AddAttribute(std::string_view Key, std::string_view Value) +{ + AttributePair* NewAttr = new (m_Arena) AttributePair(); + NewAttr->Key = m_Arena.DuplicateString(Key); + NewAttr->SetStringValue(m_Arena.DuplicateString(Value)); + NewAttr->Next = m_Attributes; + m_Attributes = NewAttr; +} + +void +Span::AddAttribute(std::string_view Key, uint64_t Value) +{ + AttributePair* NewAttr = new (m_Arena) AttributePair(); + NewAttr->Key = m_Arena.DuplicateString(Key); + NewAttr->SetNumericValue(Value); + NewAttr->Next = m_Attributes; + m_Attributes = NewAttr; +} + +void +Span::AddAttributes(const AttributeList& Attributes) +{ + for (const auto& [K, V] : Attributes) + { + AddAttribute(K, V); + } +} + +void* +Span::operator new(size_t Size, zen::MemoryArena& Arena) +{ + return Arena.Allocate(Size); +} + +void +Span::operator delete(void* Ptr, zen::MemoryArena& Arena) +{ + // This exists because operator new with placement arguments + // requires a matching operator delete, but we don't actually + // need to do anything here + ZEN_UNUSED(Ptr, Arena); +} + +////////////////////////////////////////////////////////////////////////// + +std::atomic<uint32_t> g_TraceCounter; + +TraceId +TraceId::NewTraceId() +{ + uint8_t TraceIdBytes[kSize]; + + // We use our session ID as the basis for trace IDs + const Oid SessionId = GetSessionId(); + memcpy(TraceIdBytes, SessionId.OidBits, sizeof SessionId.OidBits); + + const uint32_t TraceCounterBe = ByteSwap(g_TraceCounter.fetch_add(1)); + + memcpy(&TraceIdBytes[12], &TraceCounterBe, 4); + + return TraceId(std::span<const uint8_t, kSize>(TraceIdBytes, kSize)); +} + +////////////////////////////////////////////////////////////////////////// + +SpanId +SpanId::NewSpanId() +{ + // Just use a new OID and take the first 8 bytes. We'll probably want + // a more native solution later + + return SpanId(Oid::NewOid()); +} + +////////////////////////////////////////////////////////////////////////// + +Ref<TraceRecorder> g_TraceRecorder; + +void +SetTraceRecorder(Ref<TraceRecorder> Recorder) +{ + g_TraceRecorder = std::move(Recorder); +} + +bool +IsRecording() +{ + return !!g_TraceRecorder; +} + +////////////////////////////////////////////////////////////////////////// + +thread_local Tracer* t_Tracer; + +struct Tracer::Impl +{ + Impl() : m_PrevTracer(t_Tracer) {} + ~Impl() { t_Tracer = m_PrevTracer; } + + MemoryArena m_Arena; + Tracer* m_PrevTracer; + Span* m_SpanChain = nullptr; + std::atomic<uint64_t> m_SpanCount; +}; + +Tracer::Tracer() : m_Impl(new Impl()) +{ + t_Tracer = this; +} + +Tracer::~Tracer() +{ + if (Ref<TraceRecorder> Recorder = g_TraceRecorder) + { + const uint64_t SpanCount = m_Impl->m_SpanCount.load(); + const Span** Spans = new (m_Impl->m_Arena) const Span*[SpanCount]; + + uint64_t Index = 0; + + for (Span* CurrentSpan = m_Impl->m_SpanChain; CurrentSpan != nullptr; CurrentSpan = CurrentSpan->m_SpanChain) + { + Spans[Index++] = CurrentSpan; + } + + Recorder->RecordSpans(t_TraceState.CurrentTraceId, std::span<const Span*>(Spans, SpanCount)); + } + + delete m_Impl; +} + +Tracer* +Tracer::GetTracer() +{ + Tracer* TracerPtr = t_Tracer; + + if (!TracerPtr) + { + TracerPtr = new Tracer(); + } + + return TracerPtr; +} + +ScopedSpan +Tracer::CreateSpan(std::string_view Name) +{ + Tracer* TracerPtr = GetTracer(); + + Impl* const ImplPtr = TracerPtr->m_Impl; + + Span* NewSpan = new (ImplPtr->m_Arena) Span(ImplPtr->m_Arena, Name, ImplPtr->m_SpanChain); + + ImplPtr->m_SpanChain = NewSpan; + ImplPtr->m_SpanCount.fetch_add(1); + + return ScopedSpan(NewSpan, TracerPtr); +} + +////////////////////////////////////////////////////////////////////////// + +ScopedSpan::ScopedSpan(std::string_view Name) +{ + Tracer* TracerPtr = Tracer::GetTracer(); + + Tracer::Impl* const ImplPtr = TracerPtr->m_Impl; + + Span* NewSpan = new (ImplPtr->m_Arena) Span(ImplPtr->m_Arena, Name, ImplPtr->m_SpanChain); + + ImplPtr->m_SpanChain = NewSpan; + ImplPtr->m_SpanCount.fetch_add(1); + + m_Tracer = TracerPtr; + m_Span = NewSpan; +} + +ScopedSpan::ScopedSpan(Span* InSpan, Tracer* InTracer) : m_Tracer(InTracer), m_Span(InSpan) +{ +} + +ScopedSpan::~ScopedSpan() +{ +} + +} // namespace zen::otel + +////////////////////////////////////////////////////////////////////////// + +namespace zen::otel { + +using namespace std::literals; + +void +otlptrace_forcelink() +{ +} + +# if ZEN_WITH_TESTS + +TEST_CASE("otlp.trace") +{ + { + ScopedSpan Span = Tracer::CreateSpan("span0"); + Span->AddEvent("TestEvent1"sv); + Span->AddEvent("TestEvent2"sv); + + { + ScopedSpan Span2 = Tracer::CreateSpan("span1"); + Span2->AddEvent("TestEvent3"sv); + Span2->AddEvent("TestEvent4"sv); + } + } +} + +# endif + +} // namespace zen::otel +#endif diff --git a/src/zencore/stats.cpp b/src/zentelemetry/stats.cpp index 8a424c5ad..c67fa3c66 100644 --- a/src/zencore/stats.cpp +++ b/src/zentelemetry/stats.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "zencore/stats.h" +#include "zentelemetry/stats.h" #include <zencore/compactbinarybuilder.h> #include <zencore/intmath.h> diff --git a/src/zentelemetry/xmake.lua b/src/zentelemetry/xmake.lua new file mode 100644 index 000000000..2f90d7b90 --- /dev/null +++ b/src/zentelemetry/xmake.lua @@ -0,0 +1,10 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +target('zentelemetry') + set_kind("static") + set_group("libs") + add_headerfiles("**.h") + add_files("**.cpp") + add_includedirs("include", {public=true}) + add_deps("zencore", "protozero") + add_packages("vcpkg::robin-map", "vcpkg::spdlog") diff --git a/src/zentelemetry/zentelemetry.cpp b/src/zentelemetry/zentelemetry.cpp new file mode 100644 index 000000000..ed6ad13b9 --- /dev/null +++ b/src/zentelemetry/zentelemetry.cpp @@ -0,0 +1,19 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zentelemetry/zentelemetry.h" + +#include "zentelemetry/otlptrace.h" +#include "zentelemetry/stats.h" + +namespace zen { + +void +zentelemetry_forcelinktests() +{ + zen::stats_forcelink(); +#if ZEN_WITH_OTEL + zen::otel::otlptrace_forcelink(); +#endif +} + +} // namespace zen diff --git a/thirdparty/protozero/CHANGELOG.md b/thirdparty/protozero/CHANGELOG.md new file mode 100644 index 000000000..12231fe4d --- /dev/null +++ b/thirdparty/protozero/CHANGELOG.md @@ -0,0 +1,457 @@ + +# Changelog + +All notable changes to this project will be documented in this file. +The format is based on [Keep a Changelog](https://keepachangelog.com/) +This project adheres to [Semantic Versioning](https://semver.org/). + +## [unreleased] - + +### Added + +### Changed + +### Fixed + + +## [1.8.1] - 2025-07-15 + +### Fixed + +- Fix buffer overrun in `get_bool()` +- Fix test that checks that protozero also works with `std::string_view` + + +## [1.8.0] - 2025-01-13 + +### Changed + +- Switched to C++14 as minimum requirement. Modernised the code accordingly. +- Only the tests need the Protobuf library when building. +- Improve compatibility with various STL flavors. +- Change license formatting to get recognized by license scanning tools. +- Add lots of `const` in places where we can. Also fix some other issues + reported by clang-tidy. +- Various other code cleanups. +- Modernize documentation config, CMake config and CI workflows. + +### Fixed + +- `basic_pbf_builder` constructor can not be noexcept. + + +## [1.7.1] - 2022-01-10 + +### Changed + +- Don't build tests if the standard CMake `BUILD_TESTING` variable is set to + off. +- Now needs CMake 3.5.0 or greater. +- Update included catch2 framework to current version v2.13.8. +- Only enable clang-tidy make target if protobuf was found. +- Allow setting C++ version to compile with in CMake config. + +### Fixed + +- Fixes undefined behaviour in `float` and `double` byteswap. +- Add missing includes of "config.hpp". +- Avoid narrowing conversion by doing an explicit `static_cast`. + + +## [1.7.0] - 2020-06-08 + +### Added + +- Support for buffer types other that `std::string`. `pbf_writer` is now + just a typedef for `basic_pbf_writer<std::string>`. Other buffer types + can be used with `basic_pbf_writer`. See `doc/advanced.md` for details. + +### Changed + +- Switched to *catch2* for testing. +- Some minor tweaks. + +### Fixed + +- Removed some undefined behaviour. + + +## [1.6.8] - 2019-08-15 + +### Changed + +- Various code cleanups due to clang-tidy warnings. + +### Fixed + +- Made `data_view::compare` noexcept. + + +## [1.6.7] - 2018-02-21 + +### Fixed + +- Signed-unsigned comparison on 32 bit systems. + + +## [1.6.6] - 2018-02-20 + +### Fixed + +- Fixed several place with possible undefined behaviour. + + +## [1.6.5] - 2018-02-05 + +### Fixed + +- Avoid UB: Do not calculate pointer outside array bounds. +- Specify proto2 syntax in .proto files to appease protoc. + + +## [1.6.4] - 2018-11-08 + +### Added + +- Add function `data()` to get the not yet read data from a `pbf_reader`. +- New `add_packed_fixed()` template function for `pbf_writer`. +- New `length_of_varint()` helper function calculates how long a varint + would be for a specified value. + +### Changed + +- More consistent implementation of operators as free friend functions. + +### Fixed + +- Fixed some zigzag encoding tests on MSVC. +- Add extra cast so we do an xor with unsigned ints. +- No more bitwise operations on signed integers in varint decoder. +- No more bitwise operations on signed integers in zigzag encoder/decoder. + + +## [1.6.3] - 2018-07-17 + +### Changed + +- Moved `byteswap_inplace` functions from detail into protozero namespace. + They can be useful outsize protozero. +- More asserts and unit tests and small cleanups. + + +## [1.6.2] - 2018-03-09 + +### Changed + +- Update included catch.hpp to v1.12.0. +- Move basic unit tests into their own directory (`test/unit`). +- Improved clang-tidy config and fixed some code producing warnings. + +### Fixed + +- Buffer overflow in pbf-decoder tool. + + +## [1.6.1] - 2017-11-16 + +### Added + +- Document internal handling of varints. +- Add aliases for fixed iterators, too. + +### Changed + +- The `const_fixed_iterator` is now a random access iterator making code + using it potentially more performant (for instance when using + `std::distance`) +- Overloads `std::distance` for the varint and svarint iterators. This is + better than the workaround with the `rage_size` function used before. + +### Fixed + +- Rename `.proto` files in some tests to be unique. This solves a problem + when building with newer versions of the Google Protobuf library. +- Floating point comparisons in tests are now always correctly done using + `Approx()`. + + +## [1.6.0] - 2017-10-24 + +### Added + +- Comparison functions (<, <=, >, >=) for `data_view`. Allows use in `std::map` + for instance. +- Tool `pbf-decoder` for decoding raw messages. This has limited use for + normal users, but it can be used for fuzzing. + +### Changed + +- Protozero now uses CMake to build the tests etc. This does not affect + simple users of the library, but if you are using CMake yourself you might + want to use the `cmake/FindProtozero.cmake` module provided. The README + contains more information about build options. +- Moved `data_view` class from `types.hpp` into its own header file + `data_view.hpp`. +- Implementation of the `const_fixed_iterator` to use only a single pointer + instead of two. +- Made `operator==` and `operator!=` on `data_view` constexpr. +- The `pbf_reader` constructor taking a `std::pair` is deprecated. Use one + of the other constructors instead. + +### Fixed + +- Varints where the last byte was larger than what would fit in 64bit were + triggering undefined behaviour. This can only happen when the message + being decoded was corrupt in some way. +- Do not assert when reading too long varints for bools any more. A valid + encoder should never generate varints with more than one byte for bools, + but if they are longer that's not really a problem, so just handle it. +- Throw exception if the length of a packed repeated field of a fixed-length + type is invalid. The length must always be a multiple of the size of the + underlying type. This can only happen if the data is corrupted in some way, + a valid encoder would never generate data like this. +- Throw an exception when reading invalid tags. This can only happen if the + data is corrupted in some way, a valid encoder would never generate invalid + tags. + + +## [1.5.3] - 2017-09-22 + +### Added + +- More documentation. +- New `size()` method on iterator range used for packed repeated fields to + find out how many elements there are in the range. This is much faster + compared to the `std::difference()` call you had to do before, because the + varints don't have to be fully decoded. See [Advanced + Topics](doc/advanced.md) for details. + +### Changed + +- Updated clang-tidy settings in Makefiles and fixed a lot of minor issues + reported by clang-tidy. +- Update included catch.hpp to version 1.10.0. +- Miscellaneous code cleanups. +- Support for invalid state in `pbf_writer` and `packed_repeated_fields`. + This fixes move construction and move assignement in `pbf_writer` and + disables the copy construction and copy assignement which don't have + clear semantics. It introduces an invalid or empty state in the + `pbf_writer`, `pbf_builder`, and `packed_repeated_fields` classes used for + default-constructed, moved from, or committed objects. There is a new + `commit()` function for `pbf_writer` and the `packed_repeated_fields` which + basically does the same as the destructor but can be called explicitly. + +### Fixed + +- The `empty()` method of the iterator range now returns a `bool` instead of + a `size_t`. + + +## [1.5.2] - 2017-06-30 + +### Added + +- Add missing two-parameter version of `pbf_message::next()` function. +- Add `data_view::empty()` function. +- Add missing versions of `add_bytes()`, `add_string()`, and `add_message()` + to `pbf_builder`. + +### Changed + +- Clarify include file usage in tutorial. +- Updated included Catch unit test framework to version 1.9.6 and updated + tests to work with the current version. +- Make some constructors explicit (best practice to avoid silent conversions). + +### Fixed + +- Important bugfix in `data_view` equality operator. The equality operator is + actually never used in the protozero code itself, but users of protozero + might use it. This is a serious bug that could lead to buffer overrun type + problems. + + +## [1.5.1] - 2017-01-14 + +### Added + +- Better documentation for `tag_and_type()` in doc/advanced.md. + +### Fixed + +- Fixed broken "make doc" build. + + +## [1.5.0] - 2017-01-12 + +### Added + +- Add `add_bytes_vectored()` methods to `pbf_writer` and `pbf_builder`. This + allows single-copy scatter-gather type adding of data that has been prepared + in pieces to a protobuf message. +- New functions to check the tag and wire type at the same time: Two parameter + version of `pbf_reader::next()` and `pbf_reader::tag_and_type()` can be used + together with the free function `tag_and_type()` to easily and quickly check + that not only the tag but also the wire type is correct for a field. + +### Changed + +- `packed_field_*` classes now work with `pbf_builder`. +- Reorganized documentation. Advanced docs are now under doc/advanced.md. + +### Fixed + +- `packed_field` class is now non-copyable because data can get corrupted if + you copy it around. +- Comparison operators of `data_view` now have const& parameters. +- Make zigzag encoding/decoding functions constexpr. + + +## [1.4.5] - 2016-11-18 + +### Fixed + +- Undefined behaviour in packed fixed iterator. As a result, the macro + `PROTOZERO_DO_NOT_USE_BARE_POINTER` is not used any more. + + +## [1.4.4] - 2016-11-15 + +### Fixed + +- Byteswap implementation. + + +## [1.4.3] - 2016-11-15 + +### Fixed + +- Undefined behaviour in byte swapping code. +- Rename some parameters to avoid "shadow" warning from some compilers. + + +## [1.4.2] - 2016-08-27 + +### Fixed + +- Compile fix: Variable shadowing. + + +## [1.4.1] - 2016-08-21 + +### Fixed + +- GCC 4.8 compile fixed + +### Added + +- New ability to dynamically require the module as a node module to ease + building against from other node C++ modules. + +## [1.4.0] - 2016-07-22 + +### Changed + +- Use more efficient new `skip_varint()` function when iterating over + packed varints. +- Split `decode_varint()` function into two functions speeding up the + common case where a varint is only one byte long. +- Introduce new class `iterator_range` used instead of `std::pair` of + iterators. This way the objects can be used in range-based for loops. + Read UPGRADING.md for details. +- Introduce new class `data_view` and functions using and returning it. + Read UPGRADING.md for details. + + +## [1.3.0] - 2016-02-18 + +### Added + +- Added `config.hpp` header which now includes all the macro magic to + configure the library for different architectures etc. +- New way to create repeated packed fields without using an iterator. +- Add `rollback()` function to `pbf_writer` for "manual" rollback. + +### Changed + +- Various test and documentation cleanups. +- Rename `pbf_types.hpp` to `types.hpp`. + + +## [1.2.3] - 2015-11-30 + +### Added + +- Added `config.hpp` header which now includes all the macro magic to + configure the library for different architectures etc. + +### Fixed + +- Unaligned access to floats/doubles on some ARM architectures. + + +## [1.2.2] - 2015-10-13 + +### Fixed + +- Fix the recently broken writing of bools on big-endian architectures. + + +## [1.2.1] - 2015-10-12 + +### Fixed + +- Removed unneeded code (1-byte "swap") which lead to test failures. + + +## [1.2.0] - 2015-10-08 + +### Added + +- `pbf_message` and `pbf_builder` template classes wrapping `pbf_reader` + and `pbf_writer`, respectively. The new classes are the preferred + interface now. + +### Changed + +- Improved byte swapping operation. +- Detect some types of data corruption earlier and throw. + + +## [1.1.0] - 2015-08-22 + +### Changed + +- Make pbf reader and writer code endianess-aware. + + +[unreleased]: https://github.com/osmcode/libosmium/compare/v1.8.0...HEAD +[1.8.0]: https://github.com/osmcode/libosmium/compare/v1.7.1...v1.8.0 +[1.7.1]: https://github.com/osmcode/libosmium/compare/v1.7.0...v1.7.1 +[1.7.0]: https://github.com/osmcode/libosmium/compare/v1.6.8...v1.7.0 +[1.6.8]: https://github.com/osmcode/libosmium/compare/v1.6.7...v1.6.8 +[1.6.7]: https://github.com/osmcode/libosmium/compare/v1.6.6...v1.6.7 +[1.6.6]: https://github.com/osmcode/libosmium/compare/v1.6.5...v1.6.6 +[1.6.5]: https://github.com/osmcode/libosmium/compare/v1.6.4...v1.6.5 +[1.6.4]: https://github.com/osmcode/libosmium/compare/v1.6.3...v1.6.4 +[1.6.3]: https://github.com/osmcode/libosmium/compare/v1.6.2...v1.6.3 +[1.6.2]: https://github.com/osmcode/libosmium/compare/v1.6.1...v1.6.2 +[1.6.1]: https://github.com/osmcode/libosmium/compare/v1.6.0...v1.6.1 +[1.6.0]: https://github.com/osmcode/libosmium/compare/v1.5.3...v1.6.0 +[1.5.3]: https://github.com/osmcode/libosmium/compare/v1.5.2...v1.5.3 +[1.5.2]: https://github.com/osmcode/libosmium/compare/v1.5.1...v1.5.2 +[1.5.1]: https://github.com/osmcode/libosmium/compare/v1.5.0...v1.5.1 +[1.5.0]: https://github.com/osmcode/libosmium/compare/v1.4.5...v1.5.0 +[1.4.5]: https://github.com/osmcode/libosmium/compare/v1.4.4...v1.4.5 +[1.4.4]: https://github.com/osmcode/libosmium/compare/v1.4.3...v1.4.4 +[1.4.3]: https://github.com/osmcode/libosmium/compare/v1.4.2...v1.4.3 +[1.4.2]: https://github.com/osmcode/libosmium/compare/v1.4.1...v1.4.2 +[1.4.1]: https://github.com/osmcode/libosmium/compare/v1.4.0...v1.4.1 +[1.4.0]: https://github.com/osmcode/libosmium/compare/v1.3.0...v1.4.0 +[1.3.0]: https://github.com/osmcode/libosmium/compare/v1.2.3...v1.3.0 +[1.2.3]: https://github.com/osmcode/libosmium/compare/v1.2.2...v1.2.3 +[1.2.2]: https://github.com/osmcode/libosmium/compare/v1.2.1...v1.2.2 +[1.2.1]: https://github.com/osmcode/libosmium/compare/v1.2.0...v1.2.1 +[1.2.0]: https://github.com/osmcode/libosmium/compare/v1.1.0...v1.2.0 +[1.1.0]: https://github.com/osmcode/libosmium/compare/v1.0.0...v1.1.0 + diff --git a/thirdparty/protozero/LICENSE.from_folly b/thirdparty/protozero/LICENSE.from_folly new file mode 100644 index 000000000..f433b1a53 --- /dev/null +++ b/thirdparty/protozero/LICENSE.from_folly @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/thirdparty/protozero/LICENSE.md b/thirdparty/protozero/LICENSE.md new file mode 100644 index 000000000..5c2ce01d1 --- /dev/null +++ b/thirdparty/protozero/LICENSE.md @@ -0,0 +1,26 @@ +Copyright (C) 2022, Mapbox. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/thirdparty/protozero/README.md b/thirdparty/protozero/README.md new file mode 100644 index 000000000..a97ef3fc0 --- /dev/null +++ b/thirdparty/protozero/README.md @@ -0,0 +1,155 @@ +# protozero + +Minimalistic protocol buffer decoder and encoder in C++. + +Designed for high performance. Suitable for writing zero copy parsers and +encoders with minimal need for run-time allocation of memory. + +Low-level: this is designed to be a building block for writing a very +customized decoder for a stable protobuf schema. If your protobuf schema is +changing frequently or lazy decoding is not critical for your application then +this approach offers no value: just use the C++ API that can be generated with +the Google Protobufs `protoc` program. + +[](https://github.com/mapbox/protozero/actions/workflows/ci.yml) +[](https://ci.appveyor.com/project/Mapbox/protozero) +[](https://repology.org/metapackage/protozero) + +## Depends + +* C++14 compiler +* CMake +* Some tests depend on the Google Protobuf library, but use of Protozero + doesn't need it + + +## How it works + +The protozero code does **not** read `.proto` files used by the usual Protobuf +implementations. The developer using protozero has to manually "translate" the +`.proto` description into code. This means there is no way to access any of the +information from the `.proto` description. This results in a few restrictions: + +* The names of the fields are not available. +* Enum names are not available, you'll have to use the values they are defined + with. +* Default values are not available. +* Field types have to be hardcoded. The library does not know which types to + expect, so the user of the library has to supply the right types. Some checks + are made using `assert()`, but mostly the user has to take care of that. + +The library will make sure not to overrun the buffer it was given, but +basically all other checks have to be made in user code! + + +## Documentation + +You have to have a working knowledge of how +[protocol buffer encoding works](https://developers.google.com/protocol-buffers/docs/encoding). + +* Read the [tutorial](doc/tutorial.md) for an introduction on how to use + Protozero. +* Some advanced topics are described in an [extra document](doc/advanced.md). +* There is a table of all types and functions in the + [cheat sheet](doc/cheatsheet.md). +* Read the [upgrading instructions](UPGRADING.md) if you are upgrading from + an older version of Protozero. + +The build process will also build the Doxygen-based reference documentation if +you have Doxygen installed. Then open `doc/html/index.html` in your browser to +read it. + + +## Endianness + +Protozero uses a very simplistic test to check the byte order of the system it +compiles on. If this check is wrong, you'll get test failures. If this is the +case, please [open an issue](https://github.com/mapbox/protozero/issues) and +tell us about your system. + + +## Building tests + +Extensive tests are included. Build them using CMake: + + mkdir build + cd build + cmake .. + make + +Call `ctest` to run the tests. + +The unit and reader tests are always build, the writer tests are only build if +the Google Protobuf library is found when running CMake. + +See `test/README.md` for more details about the test. + + +## Coverage report + +To get a coverage report set `CXXFLAGS` and `LDFLAGS` before calling CMake: + + CXXFLAGS="--coverage" LDFLAGS="--coverage" cmake .. + +Then call `make` as usual and run the tests using `ctest`. + +If you are using `g++` use `gcov` to generate a report (results are in `*.gcov` +files): + + gcov -lp $(find test/ -name '*.o') + +If you are using `clang++` use `llvm-cov` instead: + + llvm-cov gcov -lp $(find test/ -name '*.o') + +If you are using `g++` you can use `gcovr` to generate nice HTML output: + + mkdir -p coverage + gcovr . -r SRCDIR --html --html-details -o coverage/index.html + +Open `coverage/index.html` in your browser to see the report. + + +## Clang-tidy + +After the CMake step, run + + make clang-tidy + +to check the code with [clang-tidy](https://clang.llvm.org/extra/clang-tidy/). +You might have to set `CLANG_TIDY` in CMake config. + + +## Cppcheck + +For extra checks with [Cppcheck](https://cppcheck.sourceforge.io/) you can, +after the CMake step, call + + make cppcheck + + +## Installation + +After the CMake step, call `make install` to install the include files in +`/usr/local/include/protozero`. + +If you are using CMake to build your own software, you can copy the file +`cmake/FindProtozero.cmake` and use it in your build. See the file for +details. + + +## Who is using Protozero? + +* [Carmen](https://github.com/mapbox/carmen-cache) +* [Libosmium](https://github.com/osmcode/libosmium) +* [Mapbox GL Native](https://github.com/mapbox/mapbox-gl-native) +* [Mapbox Vector Tile library](https://github.com/mapbox/vector-tile) +* [Mapnik](https://github.com/mapbox/mapnik-vector-tile) +* [OSRM](https://github.com/Project-OSRM/osrm-backend) +* [Tippecanoe](https://github.com/mapbox/tippecanoe) +* [Vtzero](https://github.com/mapbox/vtzero) + +Are you using Protozero? Tell us! Send a pull request with changes to this +README. + + diff --git a/thirdparty/protozero/include/protozero/basic_pbf_builder.hpp b/thirdparty/protozero/include/protozero/basic_pbf_builder.hpp new file mode 100644 index 000000000..ce7640d43 --- /dev/null +++ b/thirdparty/protozero/include/protozero/basic_pbf_builder.hpp @@ -0,0 +1,266 @@ +#ifndef PROTOZERO_BASIC_PBF_BUILDER_HPP +#define PROTOZERO_BASIC_PBF_BUILDER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file basic_pbf_builder.hpp + * + * @brief Contains the basic_pbf_builder template class. + */ + +#include "basic_pbf_writer.hpp" +#include "types.hpp" + +#include <type_traits> + +namespace protozero { + +/** + * The basic_pbf_builder is used to write PBF formatted messages into a buffer. + * It is based on the basic_pbf_writer class and has all the same methods. The + * difference is that while the pbf_writer class takes an integer tag, + * this template class takes a tag of the template type T. The idea is that + * T will be an enumeration value and this helps reduce the possibility of + * programming errors. + * + * Almost all methods in this class can throw an std::bad_alloc exception if + * the underlying buffer class wants to resize. + * + * Read the tutorial to understand how this class is used. In most cases you + * want to use the pbf_builder class which uses a std::string as buffer type. + */ +template <typename TBuffer, typename T> +class basic_pbf_builder : public basic_pbf_writer<TBuffer> { + + static_assert(std::is_same<pbf_tag_type, std::underlying_type_t<T>>::value, + "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + /// The type of messages this class will build. + using enum_type = T; + + basic_pbf_builder() = default; + + /** + * Create a builder using the given string as a data store. The object + * stores a reference to that string and adds all data to it. The string + * doesn't have to be empty. The pbf_message object will just append data. + */ + explicit basic_pbf_builder(TBuffer& data) noexcept : + basic_pbf_writer<TBuffer>{data} { + } + + /** + * Construct a pbf_builder for a submessage from the pbf_message or + * pbf_writer of the parent message. + * + * @param parent_writer The parent pbf_message or pbf_writer + * @param tag Tag of the field that will be written + */ + template <typename P> + basic_pbf_builder(basic_pbf_writer<TBuffer>& parent_writer, P tag) : + basic_pbf_writer<TBuffer>{parent_writer, pbf_tag_type(tag)} { + } + +/// @cond INTERNAL +#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \ + void add_##name(T tag, type value) { \ + basic_pbf_writer<TBuffer>::add_##name(pbf_tag_type(tag), value); \ + } + + PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double) + +#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR +/// @endcond + + /** + * Add "bytes" field to data. + * + * @param tag Tag of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + void add_bytes(T tag, const char* value, std::size_t size) { + basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value, size); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag of the field + * @param value Value to be written + */ + void add_bytes(T tag, const data_view& value) { + basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag of the field + * @param value Value to be written + */ + void add_bytes(T tag, const std::string& value) { + basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value); + } + + /** + * Add "bytes" field to data. Bytes from the value are written until + * a null byte is encountered. The null byte is not added. + * + * @param tag Tag of the field + * @param value Pointer to zero-delimited value to be written + */ + void add_bytes(T tag, const char* value) { + basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value); + } + + /** + * Add "bytes" field to data using vectored input. All the data in the + * 2nd and further arguments is "concatenated" with only a single copy + * into the final buffer. + * + * This will work with objects of any type supporting the data() and + * size() methods like std::string or protozero::data_view. + * + * Example: + * @code + * std::string data1 = "abc"; + * std::string data2 = "xyz"; + * builder.add_bytes_vectored(1, data1, data2); + * @endcode + * + * @tparam Ts List of types supporting data() and size() methods. + * @param tag Tag of the field + * @param values List of objects of types Ts with data to be appended. + */ + template <typename... Ts> + void add_bytes_vectored(T tag, Ts&&... values) { + basic_pbf_writer<TBuffer>::add_bytes_vectored(pbf_tag_type(tag), std::forward<Ts>(values)...); + } + + /** + * Add "string" field to data. + * + * @param tag Tag of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + void add_string(T tag, const char* value, std::size_t size) { + basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value, size); + } + + /** + * Add "string" field to data. + * + * @param tag Tag of the field + * @param value Value to be written + */ + void add_string(T tag, const data_view& value) { + basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value); + } + + /** + * Add "string" field to data. + * + * @param tag Tag of the field + * @param value Value to be written + */ + void add_string(T tag, const std::string& value) { + basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value); + } + + /** + * Add "string" field to data. Bytes from the value are written until + * a null byte is encountered. The null byte is not added. + * + * @param tag Tag of the field + * @param value Pointer to value to be written + */ + void add_string(T tag, const char* value) { + basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value); + } + + /** + * Add "message" field to data. + * + * @param tag Tag of the field + * @param value Pointer to message to be written + * @param size Length of the message + */ + void add_message(T tag, const char* value, std::size_t size) { + basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value, size); + } + + /** + * Add "message" field to data. + * + * @param tag Tag of the field + * @param value Value to be written. The value must be a complete message. + */ + void add_message(T tag, const data_view& value) { + basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value); + } + + /** + * Add "message" field to data. + * + * @param tag Tag of the field + * @param value Value to be written. The value must be a complete message. + */ + void add_message(T tag, const std::string& value) { + basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value); + } + +/// @cond INTERNAL +#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \ + template <typename InputIterator> \ + void add_packed_##name(T tag, InputIterator first, InputIterator last) { \ + basic_pbf_writer<TBuffer>::add_packed_##name(pbf_tag_type(tag), first, last); \ + } + + PROTOZERO_WRITER_WRAP_ADD_PACKED(bool) + PROTOZERO_WRITER_WRAP_ADD_PACKED(enum) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(float) + PROTOZERO_WRITER_WRAP_ADD_PACKED(double) + +#undef PROTOZERO_WRITER_WRAP_ADD_PACKED +/// @endcond + +}; // class basic_pbf_builder + +} // end namespace protozero + +#endif // PROTOZERO_BASIC_PBF_BUILDER_HPP diff --git a/thirdparty/protozero/include/protozero/basic_pbf_writer.hpp b/thirdparty/protozero/include/protozero/basic_pbf_writer.hpp new file mode 100644 index 000000000..c26ba67cd --- /dev/null +++ b/thirdparty/protozero/include/protozero/basic_pbf_writer.hpp @@ -0,0 +1,1054 @@ +#ifndef PROTOZERO_BASIC_PBF_WRITER_HPP +#define PROTOZERO_BASIC_PBF_WRITER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file basic_pbf_writer.hpp + * + * @brief Contains the basic_pbf_writer template class. + */ + +#include "buffer_tmpl.hpp" +#include "config.hpp" +#include "data_view.hpp" +#include "types.hpp" +#include "varint.hpp" + +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN +# include <protozero/byteswap.hpp> +#endif + +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <initializer_list> +#include <iterator> +#include <limits> +#include <string> +#include <utility> + +namespace protozero { + +namespace detail { + + template <typename B, typename T> class packed_field_varint; + template <typename B, typename T> class packed_field_svarint; + template <typename B, typename T> class packed_field_fixed; + +} // end namespace detail + +/** + * The basic_pbf_writer is used to write PBF formatted messages into a buffer. + * + * This uses TBuffer as the type for the underlaying buffer. In typical uses + * this is std::string, but you can use a different type that must support + * the right interface. Please see the documentation for details. + * + * Almost all methods in this class can throw an std::bad_alloc exception if + * the underlying buffer class wants to resize. + */ +template <typename TBuffer> +class basic_pbf_writer { + + // A pointer to a buffer holding the data already written to the PBF + // message. For default constructed writers or writers that have been + // rolled back, this is a nullptr. + TBuffer* m_data = nullptr; + + // A pointer to a parent writer object if this is a submessage. If this + // is a top-level writer, it is a nullptr. + basic_pbf_writer* m_parent_writer = nullptr; + + // This is usually 0. If there is an open submessage, this is set in the + // parent to the rollback position, ie. the last position before the + // submessage was started. This is the position where the header of the + // submessage starts. + std::size_t m_rollback_pos = 0; + + // This is usually 0. If there is an open submessage, this is set in the + // parent to the position where the data of the submessage is written to. + std::size_t m_pos = 0; + + void add_varint(uint64_t value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage"); + protozero_assert(m_data); + add_varint_to_buffer(m_data, value); + } + + void add_field(pbf_tag_type tag, pbf_wire_type type) { + protozero_assert(((tag > 0 && tag < 19000) || (tag > 19999 && tag <= ((1U << 29U) - 1))) && "tag out of range"); + const uint32_t b = (tag << 3U) | static_cast<uint32_t>(type); + add_varint(b); + } + + void add_tagged_varint(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::varint); + add_varint(value); + } + + template <typename T> + void add_fixed(T value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage"); + protozero_assert(m_data); +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN + byteswap_inplace(&value); +#endif + buffer_customization<TBuffer>::append(m_data, reinterpret_cast<const char*>(&value), sizeof(T)); + } + + template <typename T, typename It> + void add_packed_fixed(pbf_tag_type tag, It first, It last, std::input_iterator_tag /*unused*/) { // NOLINT(performance-unnecessary-value-param) + if (first == last) { + return; + } + + basic_pbf_writer sw{*this, tag}; + + while (first != last) { + sw.add_fixed<T>(*first++); + } + } + + template <typename T, typename It> + void add_packed_fixed(pbf_tag_type tag, It first, It last, std::forward_iterator_tag /*unused*/) { // NOLINT(performance-unnecessary-value-param) + if (first == last) { + return; + } + + const auto length = std::distance(first, last); + add_length_varint(tag, sizeof(T) * pbf_length_type(length)); + reserve(sizeof(T) * std::size_t(length)); + + while (first != last) { + add_fixed<T>(*first++); + } + } + + template <typename It> + void add_packed_varint(pbf_tag_type tag, It first, It last) { // NOLINT(performance-unnecessary-value-param) + if (first == last) { + return; + } + + basic_pbf_writer sw{*this, tag}; + + while (first != last) { + sw.add_varint(uint64_t(*first++)); + } + } + + template <typename It> + void add_packed_svarint(pbf_tag_type tag, It first, It last) { // NOLINT(performance-unnecessary-value-param) + if (first == last) { + return; + } + + basic_pbf_writer sw{*this, tag}; + + while (first != last) { + sw.add_varint(encode_zigzag64(*first++)); + } + } + + // The number of bytes to reserve for the varint holding the length of + // a length-delimited field. The length has to fit into pbf_length_type, + // and a varint needs 8 bit for every 7 bit. + enum : int { + reserve_bytes = (sizeof(pbf_length_type) * 8 / 7) + 1 + }; + + // If m_rollpack_pos is set to this special value, it means that when + // the submessage is closed, nothing needs to be done, because the length + // of the submessage has already been written correctly. + enum : std::size_t { + size_is_known = std::numeric_limits<std::size_t>::max() + }; + + void open_submessage(pbf_tag_type tag, std::size_t size) { + protozero_assert(m_pos == 0); + protozero_assert(m_data); + if (size == 0) { + m_rollback_pos = buffer_customization<TBuffer>::size(m_data); + add_field(tag, pbf_wire_type::length_delimited); + buffer_customization<TBuffer>::append_zeros(m_data, std::size_t(reserve_bytes)); + } else { + m_rollback_pos = size_is_known; + add_length_varint(tag, static_cast<pbf_length_type>(size)); + reserve(size); + } + m_pos = buffer_customization<TBuffer>::size(m_data); + } + + void rollback_submessage() { + protozero_assert(m_pos != 0); + protozero_assert(m_rollback_pos != size_is_known); + protozero_assert(m_data); + buffer_customization<TBuffer>::resize(m_data, m_rollback_pos); + m_pos = 0; + } + + void commit_submessage() { + protozero_assert(m_pos != 0); + protozero_assert(m_rollback_pos != size_is_known); + protozero_assert(m_data); + const auto length = pbf_length_type(buffer_customization<TBuffer>::size(m_data) - m_pos); + + protozero_assert(buffer_customization<TBuffer>::size(m_data) >= m_pos - reserve_bytes); + const auto n = add_varint_to_buffer(buffer_customization<TBuffer>::at_pos(m_data, m_pos - reserve_bytes), length); + + buffer_customization<TBuffer>::erase_range(m_data, m_pos - reserve_bytes + n, m_pos); + m_pos = 0; + } + + void close_submessage() { + protozero_assert(m_data); + if (m_pos == 0 || m_rollback_pos == size_is_known) { + return; + } + if (buffer_customization<TBuffer>::size(m_data) - m_pos == 0) { + rollback_submessage(); + } else { + commit_submessage(); + } + } + + void add_length_varint(pbf_tag_type tag, pbf_length_type length) { + add_field(tag, pbf_wire_type::length_delimited); + add_varint(length); + } + +public: + + /** + * Create a writer using the specified buffer as a data store. The + * basic_pbf_writer stores a pointer to that buffer and adds all data to + * it. The buffer doesn't have to be empty. The basic_pbf_writer will just + * append data. + */ + explicit basic_pbf_writer(TBuffer& buffer) noexcept : + m_data{&buffer} { + } + + /** + * Create a writer without a data store. In this form the writer can not + * be used! + */ + basic_pbf_writer() noexcept = default; + + /** + * Construct a basic_pbf_writer for a submessage from the basic_pbf_writer + * of the parent message. + * + * @param parent_writer The basic_pbf_writer + * @param tag Tag (field number) of the field that will be written + * @param size Optional size of the submessage in bytes (use 0 for unknown). + * Setting this allows some optimizations but is only possible in + * a few very specific cases. + */ + basic_pbf_writer(basic_pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size = 0) : + m_data{parent_writer.m_data}, + m_parent_writer{&parent_writer} { + m_parent_writer->open_submessage(tag, size); + } + + /// A basic_pbf_writer object can not be copied + basic_pbf_writer(const basic_pbf_writer&) = delete; + + /// A basic_pbf_writer object can not be copied + basic_pbf_writer& operator=(const basic_pbf_writer&) = delete; + + /** + * A basic_pbf_writer object can be moved. After this the other + * basic_pbf_writer will be invalid. + */ + basic_pbf_writer(basic_pbf_writer&& other) noexcept : + m_data{other.m_data}, + m_parent_writer{other.m_parent_writer}, + m_rollback_pos{other.m_rollback_pos}, + m_pos{other.m_pos} { + other.m_data = nullptr; + other.m_parent_writer = nullptr; + other.m_rollback_pos = 0; + other.m_pos = 0; + } + + /** + * A basic_pbf_writer object can be moved. After this the other + * basic_pbf_writer will be invalid. + */ + basic_pbf_writer& operator=(basic_pbf_writer&& other) noexcept { + m_data = other.m_data; + m_parent_writer = other.m_parent_writer; + m_rollback_pos = other.m_rollback_pos; + m_pos = other.m_pos; + other.m_data = nullptr; + other.m_parent_writer = nullptr; + other.m_rollback_pos = 0; + other.m_pos = 0; + return *this; + } + + ~basic_pbf_writer() noexcept { + try { + if (m_parent_writer != nullptr) { + m_parent_writer->close_submessage(); + } + } catch (...) { + // This try/catch is used to make the destructor formally noexcept. + // close_submessage() is not noexcept, but will not throw the way + // it is called here, so we are good. But to be paranoid, call... + std::terminate(); + } + } + + /** + * Check if this writer is valid. A writer is invalid if it was default + * constructed, moved from, or if commit() has been called on it. + * Otherwise it is valid. + */ + bool valid() const noexcept { + return m_data != nullptr; + } + + /** + * Swap the contents of this object with the other. + * + * @param other Other object to swap data with. + */ + void swap(basic_pbf_writer& other) noexcept { + using std::swap; + swap(m_data, other.m_data); + swap(m_parent_writer, other.m_parent_writer); + swap(m_rollback_pos, other.m_rollback_pos); + swap(m_pos, other.m_pos); + } + + /** + * Reserve size bytes in the underlying message store in addition to + * whatever the message store already holds. So unlike + * the `std::string::reserve()` method this is not an absolute size, + * but additional memory that should be reserved. + * + * @param size Number of bytes to reserve in underlying message store. + */ + void reserve(std::size_t size) { + protozero_assert(m_data); + buffer_customization<TBuffer>::reserve_additional(m_data, size); + } + + /** + * Commit this submessage. This does the same as when the basic_pbf_writer + * goes out of scope and is destructed. + * + * @pre Must be a basic_pbf_writer of a submessage, ie one opened with the + * basic_pbf_writer constructor taking a parent message. + * @post The basic_pbf_writer is invalid and can't be used any more. + */ + void commit() { + protozero_assert(m_parent_writer && "you can't call commit() on a basic_pbf_writer without a parent"); + protozero_assert(m_pos == 0 && "you can't call commit() on a basic_pbf_writer that has an open nested submessage"); + m_parent_writer->close_submessage(); + m_parent_writer = nullptr; + m_data = nullptr; + } + + /** + * Cancel writing of this submessage. The complete submessage will be + * removed as if it was never created and no fields were added. + * + * @pre Must be a basic_pbf_writer of a submessage, ie one opened with the + * basic_pbf_writer constructor taking a parent message. + * @post The basic_pbf_writer is invalid and can't be used any more. + */ + void rollback() { + protozero_assert(m_parent_writer && "you can't call rollback() on a basic_pbf_writer without a parent"); + protozero_assert(m_pos == 0 && "you can't call rollback() on a basic_pbf_writer that has an open nested submessage"); + m_parent_writer->rollback_submessage(); + m_parent_writer = nullptr; + m_data = nullptr; + } + + ///@{ + /** + * @name Scalar field writer functions + */ + + /** + * Add "bool" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_bool(pbf_tag_type tag, bool value) { + add_field(tag, pbf_wire_type::varint); + protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage"); + protozero_assert(m_data); + m_data->push_back(static_cast<char>(value)); + } + + /** + * Add "enum" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_enum(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, static_cast<uint64_t>(value)); + } + + /** + * Add "int32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_int32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, static_cast<uint64_t>(value)); + } + + /** + * Add "sint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_sint32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, encode_zigzag32(value)); + } + + /** + * Add "uint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_uint32(pbf_tag_type tag, uint32_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "int64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_int64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, static_cast<uint64_t>(value)); + } + + /** + * Add "sint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_sint64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, encode_zigzag64(value)); + } + + /** + * Add "uint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_uint64(pbf_tag_type tag, uint64_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "fixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_fixed32(pbf_tag_type tag, uint32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed<uint32_t>(value); + } + + /** + * Add "sfixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_sfixed32(pbf_tag_type tag, int32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed<int32_t>(value); + } + + /** + * Add "fixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_fixed64(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed<uint64_t>(value); + } + + /** + * Add "sfixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_sfixed64(pbf_tag_type tag, int64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed<int64_t>(value); + } + + /** + * Add "float" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_float(pbf_tag_type tag, float value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed<float>(value); + } + + /** + * Add "double" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_double(pbf_tag_type tag, double value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed<double>(value); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + void add_bytes(pbf_tag_type tag, const char* value, std::size_t size) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage"); + protozero_assert(m_data); + protozero_assert(size <= std::numeric_limits<pbf_length_type>::max()); + add_length_varint(tag, static_cast<pbf_length_type>(size)); + buffer_customization<TBuffer>::append(m_data, value, size); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_bytes(pbf_tag_type tag, const data_view& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_bytes(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "bytes" field to data. Bytes from the value are written until + * a null byte is encountered. The null byte is not added. + * + * @param tag Tag (field number) of the field + * @param value Pointer to zero-delimited value to be written + */ + void add_bytes(pbf_tag_type tag, const char* value) { + add_bytes(tag, value, std::strlen(value)); + } + + /** + * Add "bytes" field to data using vectored input. All the data in the + * 2nd and further arguments is "concatenated" with only a single copy + * into the final buffer. + * + * This will work with objects of any type supporting the data() and + * size() methods like std::string or protozero::data_view. + * + * Example: + * @code + * std::string data1 = "abc"; + * std::string data2 = "xyz"; + * writer.add_bytes_vectored(1, data1, data2); + * @endcode + * + * @tparam Ts List of types supporting data() and size() methods. + * @param tag Tag (field number) of the field + * @param values List of objects of types Ts with data to be appended. + */ + template <typename... Ts> + void add_bytes_vectored(pbf_tag_type tag, Ts&&... values) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage"); + protozero_assert(m_data); + size_t sum_size = 0; + (void)std::initializer_list<size_t>{sum_size += values.size()...}; + protozero_assert(sum_size <= std::numeric_limits<pbf_length_type>::max()); + add_length_varint(tag, static_cast<pbf_length_type>(sum_size)); + buffer_customization<TBuffer>::reserve_additional(m_data, sum_size); + (void)std::initializer_list<int>{(buffer_customization<TBuffer>::append(m_data, values.data(), values.size()), 0)...}; + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + void add_string(pbf_tag_type tag, const char* value, std::size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_string(pbf_tag_type tag, const data_view& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + void add_string(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. Bytes from the value are written until + * a null byte is encountered. The null byte is not added. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + */ + void add_string(pbf_tag_type tag, const char* value) { + add_bytes(tag, value, std::strlen(value)); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to message to be written + * @param size Length of the message + */ + void add_message(pbf_tag_type tag, const char* value, std::size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written. The value must be a complete message. + */ + void add_message(pbf_tag_type tag, const data_view& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written. The value must be a complete message. + */ + void add_message(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + ///@} + + ///@{ + /** + * @name Repeated packed field writer functions + */ + + /** + * Add "repeated packed bool" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to bool. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_bool(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed enum" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_enum(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int32" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_int32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint32" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_sint32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint32" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_uint32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int64" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_int64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint64" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_sint64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint64" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_uint64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_varint(tag, first, last); + } + + /** + * Add a "repeated packed" fixed-size field to data. The following + * fixed-size fields are available: + * + * uint32_t -> repeated packed fixed32 + * int32_t -> repeated packed sfixed32 + * uint64_t -> repeated packed fixed64 + * int64_t -> repeated packed sfixed64 + * double -> repeated packed double + * float -> repeated packed float + * + * @tparam ValueType One of the following types: (u)int32/64_t, double, float. + * @tparam InputIterator A type satisfying the InputIterator concept. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename ValueType, typename InputIterator> + void add_packed_fixed(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + static_assert(std::is_same<ValueType, uint32_t>::value || + std::is_same<ValueType, int32_t>::value || + std::is_same<ValueType, int64_t>::value || + std::is_same<ValueType, uint64_t>::value || + std::is_same<ValueType, double>::value || + std::is_same<ValueType, float>::value, "Only some types are allowed"); + add_packed_fixed<ValueType, InputIterator>(tag, first, last, + typename std::iterator_traits<InputIterator>::iterator_category{}); + } + + /** + * Add "repeated packed fixed32" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_fixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_fixed<uint32_t, InputIterator>(tag, first, last, + typename std::iterator_traits<InputIterator>::iterator_category{}); + } + + /** + * Add "repeated packed sfixed32" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_sfixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_fixed<int32_t, InputIterator>(tag, first, last, + typename std::iterator_traits<InputIterator>::iterator_category{}); + } + + /** + * Add "repeated packed fixed64" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_fixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_fixed<uint64_t, InputIterator>(tag, first, last, + typename std::iterator_traits<InputIterator>::iterator_category{}); + } + + /** + * Add "repeated packed sfixed64" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_sfixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_fixed<int64_t, InputIterator>(tag, first, last, + typename std::iterator_traits<InputIterator>::iterator_category{}); + } + + /** + * Add "repeated packed float" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to float. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_float(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_fixed<float, InputIterator>(tag, first, last, + typename std::iterator_traits<InputIterator>::iterator_category{}); + } + + /** + * Add "repeated packed double" field to data. + * + * @tparam InputIterator A type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to double. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template <typename InputIterator> + void add_packed_double(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param) + add_packed_fixed<double, InputIterator>(tag, first, last, + typename std::iterator_traits<InputIterator>::iterator_category{}); + } + + ///@} + + template <typename B, typename T> friend class detail::packed_field_varint; + template <typename B, typename T> friend class detail::packed_field_svarint; + template <typename B, typename T> friend class detail::packed_field_fixed; + +}; // class basic_pbf_writer + +/** + * Swap two basic_pbf_writer objects. + * + * @param lhs First object. + * @param rhs Second object. + */ +template <typename TBuffer> +inline void swap(basic_pbf_writer<TBuffer>& lhs, basic_pbf_writer<TBuffer>& rhs) noexcept { + lhs.swap(rhs); +} + +namespace detail { + + template <typename TBuffer> + class packed_field { + + basic_pbf_writer<TBuffer> m_writer{}; + + public: + + packed_field(const packed_field&) = delete; + packed_field& operator=(const packed_field&) = delete; + + packed_field(packed_field&&) noexcept = default; + packed_field& operator=(packed_field&&) noexcept = default; + + packed_field() = default; + + packed_field(basic_pbf_writer<TBuffer>& parent_writer, pbf_tag_type tag) : + m_writer{parent_writer, tag} { + } + + packed_field(basic_pbf_writer<TBuffer>& parent_writer, pbf_tag_type tag, std::size_t size) : + m_writer{parent_writer, tag, size} { + } + + ~packed_field() noexcept = default; + + bool valid() const noexcept { + return m_writer.valid(); + } + + void commit() { + m_writer.commit(); + } + + void rollback() { + m_writer.rollback(); + } + + basic_pbf_writer<TBuffer>& writer() noexcept { + return m_writer; + } + + }; // class packed_field + + template <typename TBuffer, typename T> + class packed_field_fixed : public packed_field<TBuffer> { + + public: + + packed_field_fixed() : + packed_field<TBuffer>{} { + } + + template <typename P> + packed_field_fixed(basic_pbf_writer<TBuffer>& parent_writer, P tag) : + packed_field<TBuffer>{parent_writer, static_cast<pbf_tag_type>(tag)} { + } + + template <typename P> + packed_field_fixed(basic_pbf_writer<TBuffer>& parent_writer, P tag, std::size_t size) : + packed_field<TBuffer>{parent_writer, static_cast<pbf_tag_type>(tag), size * sizeof(T)} { + } + + void add_element(T value) { + this->writer().template add_fixed<T>(value); + } + + }; // class packed_field_fixed + + template <typename TBuffer, typename T> + class packed_field_varint : public packed_field<TBuffer> { + + public: + + packed_field_varint() : + packed_field<TBuffer>{} { + } + + template <typename P> + packed_field_varint(basic_pbf_writer<TBuffer>& parent_writer, P tag) : + packed_field<TBuffer>{parent_writer, static_cast<pbf_tag_type>(tag)} { + } + + void add_element(T value) { + this->writer().add_varint(uint64_t(value)); + } + + }; // class packed_field_varint + + template <typename TBuffer, typename T> + class packed_field_svarint : public packed_field<TBuffer> { + + public: + + packed_field_svarint() : + packed_field<TBuffer>{} { + } + + template <typename P> + packed_field_svarint(basic_pbf_writer<TBuffer>& parent_writer, P tag) : + packed_field<TBuffer>{parent_writer, static_cast<pbf_tag_type>(tag)} { + } + + void add_element(T value) { + this->writer().add_varint(encode_zigzag64(value)); + } + + }; // class packed_field_svarint + +} // end namespace detail + +} // end namespace protozero + +#endif // PROTOZERO_BASIC_PBF_WRITER_HPP diff --git a/thirdparty/protozero/include/protozero/buffer_fixed.hpp b/thirdparty/protozero/include/protozero/buffer_fixed.hpp new file mode 100644 index 000000000..b2e6d1d27 --- /dev/null +++ b/thirdparty/protozero/include/protozero/buffer_fixed.hpp @@ -0,0 +1,222 @@ +#ifndef PROTOZERO_BUFFER_FIXED_HPP +#define PROTOZERO_BUFFER_FIXED_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file buffer_fixed.hpp + * + * @brief Contains the fixed_size_buffer_adaptor class. + */ + +#include "buffer_tmpl.hpp" +#include "config.hpp" + +#include <algorithm> +#include <cstddef> +#include <iterator> +#include <stdexcept> + +namespace protozero { + +/** + * This class can be used instead of std::string if you want to create a + * vector tile in a fixed-size buffer. Any operation that needs more space + * than is available will fail with a std::length_error exception. + */ +class fixed_size_buffer_adaptor { + + char* m_data; + std::size_t m_capacity; + std::size_t m_size = 0; + +public: + + /// @cond usual container typedefs not documented + + using size_type = std::size_t; + + using value_type = char; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + + using iterator = pointer; + using const_iterator = const_pointer; + + /// @endcond + + /** + * Constructor. + * + * @param data Pointer to some memory allocated for the buffer. + * @param capacity Number of bytes available. + */ + fixed_size_buffer_adaptor(char* data, std::size_t capacity) noexcept : + m_data(data), + m_capacity(capacity) { + } + + /** + * Constructor. + * + * @param container Some container class supporting the member functions + * data() and size(). + */ + template <typename T> + explicit fixed_size_buffer_adaptor(T& container) : + m_data(container.data()), + m_capacity(container.size()) { + } + + /// Returns a pointer to the data in the buffer. + const char* data() const noexcept { + return m_data; + } + + /// Returns a pointer to the data in the buffer. + char* data() noexcept { + return m_data; + } + + /// The capacity this buffer was created with. + std::size_t capacity() const noexcept { + return m_capacity; + } + + /// The number of bytes used in the buffer. Always <= capacity(). + std::size_t size() const noexcept { + return m_size; + } + + /// Return iterator to beginning of data. + char* begin() noexcept { + return m_data; + } + + /// Return iterator to beginning of data. + const char* begin() const noexcept { + return m_data; + } + + /// Return iterator to beginning of data. + const char* cbegin() const noexcept { + return m_data; + } + + /// Return iterator to end of data. + char* end() noexcept { + return m_data + m_size; + } + + /// Return iterator to end of data. + const char* end() const noexcept { + return m_data + m_size; + } + + /// Return iterator to end of data. + const char* cend() const noexcept { + return m_data + m_size; + } + +/// @cond INTERNAL + + // Do not rely on anything beyond this point + + void append(const char* data, std::size_t count) { + if (m_size + count > m_capacity) { + throw std::length_error{"fixed size data store exhausted"}; + } + std::copy_n(data, count, m_data + m_size); + m_size += count; + } + + void append_zeros(std::size_t count) { + if (m_size + count > m_capacity) { + throw std::length_error{"fixed size data store exhausted"}; + } + std::fill_n(m_data + m_size, count, '\0'); + m_size += count; + } + + void resize(std::size_t size) { + protozero_assert(size < m_size); + if (size > m_capacity) { + throw std::length_error{"fixed size data store exhausted"}; + } + m_size = size; + } + + void erase_range(std::size_t from, std::size_t to) { + protozero_assert(from <= m_size); + protozero_assert(to <= m_size); + protozero_assert(from < to); + std::copy(m_data + to, m_data + m_size, m_data + from); + m_size -= (to - from); + } + + char* at_pos(std::size_t pos) { + protozero_assert(pos <= m_size); + return m_data + pos; + } + + void push_back(char ch) { + if (m_size >= m_capacity) { + throw std::length_error{"fixed size data store exhausted"}; + } + m_data[m_size++] = ch; + } +/// @endcond + +}; // class fixed_size_buffer_adaptor + +/// @cond INTERNAL +template <> +struct buffer_customization<fixed_size_buffer_adaptor> { + + static std::size_t size(const fixed_size_buffer_adaptor* buffer) noexcept { + return buffer->size(); + } + + static void append(fixed_size_buffer_adaptor* buffer, const char* data, std::size_t count) { + buffer->append(data, count); + } + + static void append_zeros(fixed_size_buffer_adaptor* buffer, std::size_t count) { + buffer->append_zeros(count); + } + + static void resize(fixed_size_buffer_adaptor* buffer, std::size_t size) { + buffer->resize(size); + } + + static void reserve_additional(fixed_size_buffer_adaptor* /*buffer*/, std::size_t /*size*/) { + /* nothing to be done for fixed-size buffers */ + } + + static void erase_range(fixed_size_buffer_adaptor* buffer, std::size_t from, std::size_t to) { + buffer->erase_range(from, to); + } + + static char* at_pos(fixed_size_buffer_adaptor* buffer, std::size_t pos) { + return buffer->at_pos(pos); + } + + static void push_back(fixed_size_buffer_adaptor* buffer, char ch) { + buffer->push_back(ch); + } + +}; +/// @endcond + +} // namespace protozero + +#endif // PROTOZERO_BUFFER_FIXED_HPP diff --git a/thirdparty/protozero/include/protozero/buffer_string.hpp b/thirdparty/protozero/include/protozero/buffer_string.hpp new file mode 100644 index 000000000..c620879e4 --- /dev/null +++ b/thirdparty/protozero/include/protozero/buffer_string.hpp @@ -0,0 +1,78 @@ +#ifndef PROTOZERO_BUFFER_STRING_HPP +#define PROTOZERO_BUFFER_STRING_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file buffer_string.hpp + * + * @brief Contains the customization points for buffer implementation based + * on std::string + */ + +#include "buffer_tmpl.hpp" +#include "config.hpp" + +#include <cstddef> +#include <iterator> +#include <string> + +namespace protozero { + +// Implementation of buffer customizations points for std::string + +/// @cond INTERNAL +template <> +struct buffer_customization<std::string> { + + static std::size_t size(const std::string* buffer) noexcept { + return buffer->size(); + } + + static void append(std::string* buffer, const char* data, std::size_t count) { + buffer->append(data, count); + } + + static void append_zeros(std::string* buffer, std::size_t count) { + buffer->append(count, '\0'); + } + + static void resize(std::string* buffer, std::size_t size) { + protozero_assert(size < buffer->size()); + buffer->resize(size); + } + + static void reserve_additional(std::string* buffer, std::size_t size) { + buffer->reserve(buffer->size() + size); + } + + static void erase_range(std::string* buffer, std::size_t from, std::size_t to) { + protozero_assert(from <= buffer->size()); + protozero_assert(to <= buffer->size()); + protozero_assert(from <= to); + buffer->erase(buffer->begin() + static_cast<std::string::difference_type>(from), + buffer->begin() + static_cast<std::string::difference_type>(to)); + } + + static char* at_pos(std::string* buffer, std::size_t pos) { + protozero_assert(pos <= buffer->size()); + return (&*buffer->begin()) + pos; + } + + static void push_back(std::string* buffer, char ch) { + buffer->push_back(ch); + } + +}; +/// @endcond + +} // namespace protozero + +#endif // PROTOZERO_BUFFER_STRING_HPP diff --git a/thirdparty/protozero/include/protozero/buffer_tmpl.hpp b/thirdparty/protozero/include/protozero/buffer_tmpl.hpp new file mode 100644 index 000000000..ac223996d --- /dev/null +++ b/thirdparty/protozero/include/protozero/buffer_tmpl.hpp @@ -0,0 +1,113 @@ +#ifndef PROTOZERO_BUFFER_TMPL_HPP +#define PROTOZERO_BUFFER_TMPL_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file buffer_tmpl.hpp + * + * @brief Contains the customization points for buffer implementations. + */ + +#include <cstddef> +#include <iterator> +#include <string> + +namespace protozero { + +// Implementation of buffer customizations points for std::string + +/// @cond INTERNAL +template <typename T> +struct buffer_customization { + + /** + * Get the number of bytes currently used in the buffer. + * + * @param buffer Pointer to the buffer. + * @returns number of bytes used in the buffer. + */ + static std::size_t size(const std::string* buffer); + + /** + * Append count bytes from data to the buffer. + * + * @param buffer Pointer to the buffer. + * @param data Pointer to the data. + * @param count Number of bytes to be added to the buffer. + */ + static void append(std::string* buffer, const char* data, std::size_t count); + + /** + * Append count zero bytes to the buffer. + * + * @param buffer Pointer to the buffer. + * @param count Number of bytes to be added to the buffer. + */ + static void append_zeros(std::string* buffer, std::size_t count); + + /** + * Shrink the buffer to the specified size. The new size will always be + * smaller than the current size. + * + * @param buffer Pointer to the buffer. + * @param size New size of the buffer. + * + * @pre size < current size of buffer + */ + static void resize(std::string* buffer, std::size_t size); + + /** + * Reserve an additional size bytes for use in the buffer. This is used for + * variable-sized buffers to tell the buffer implementation that soon more + * memory will be used. The implementation can ignore this. + * + * @param buffer Pointer to the buffer. + * @param size Number of bytes to reserve. + */ + static void reserve_additional(std::string* buffer, std::size_t size); + + /** + * Delete data from the buffer. This must move back the data after the + * part being deleted and resize the buffer accordingly. + * + * @param buffer Pointer to the buffer. + * @param from Offset into the buffer where we want to erase from. + * @param to Offset into the buffer one past the last byte we want to erase. + * + * @pre from, to <= size of the buffer, from < to + */ + static void erase_range(std::string* buffer, std::size_t from, std::size_t to); + + /** + * Return a pointer to the memory at the specified position in the buffer. + * + * @param buffer Pointer to the buffer. + * @param pos The position in the buffer. + * @returns pointer to the memory in the buffer at the specified position. + * + * @pre pos <= size of the buffer + */ + static char* at_pos(std::string* buffer, std::size_t pos); + + /** + * Add a char to the buffer incrementing the number of chars in the buffer. + * + * @param buffer Pointer to the buffer. + * @param ch The character to add. + */ + static void push_back(std::string* buffer, char ch); + +}; +/// @endcond + +} // namespace protozero + +#endif // PROTOZERO_BUFFER_TMPL_HPP diff --git a/thirdparty/protozero/include/protozero/buffer_vector.hpp b/thirdparty/protozero/include/protozero/buffer_vector.hpp new file mode 100644 index 000000000..c163300c5 --- /dev/null +++ b/thirdparty/protozero/include/protozero/buffer_vector.hpp @@ -0,0 +1,78 @@ +#ifndef PROTOZERO_BUFFER_VECTOR_HPP +#define PROTOZERO_BUFFER_VECTOR_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file buffer_vector.hpp + * + * @brief Contains the customization points for buffer implementation based + * on std::vector<char> + */ + +#include "buffer_tmpl.hpp" +#include "config.hpp" + +#include <cstddef> +#include <iterator> +#include <vector> + +namespace protozero { + +// Implementation of buffer customizations points for std::vector<char> + +/// @cond INTERNAL +template <> +struct buffer_customization<std::vector<char>> { + + static std::size_t size(const std::vector<char>* buffer) noexcept { + return buffer->size(); + } + + static void append(std::vector<char>* buffer, const char* data, std::size_t count) { + buffer->insert(buffer->end(), data, data + count); + } + + static void append_zeros(std::vector<char>* buffer, std::size_t count) { + buffer->insert(buffer->end(), count, '\0'); + } + + static void resize(std::vector<char>* buffer, std::size_t size) { + protozero_assert(size < buffer->size()); + buffer->resize(size); + } + + static void reserve_additional(std::vector<char>* buffer, std::size_t size) { + buffer->reserve(buffer->size() + size); + } + + static void erase_range(std::vector<char>* buffer, std::size_t from, std::size_t to) { + protozero_assert(from <= buffer->size()); + protozero_assert(to <= buffer->size()); + protozero_assert(from <= to); + buffer->erase(std::next(buffer->begin(), static_cast<std::string::iterator::difference_type>(from)), + std::next(buffer->begin(), static_cast<std::string::iterator::difference_type>(to))); + } + + static char* at_pos(std::vector<char>* buffer, std::size_t pos) { + protozero_assert(pos <= buffer->size()); + return (&*buffer->begin()) + pos; + } + + static void push_back(std::vector<char>* buffer, char ch) { + buffer->push_back(ch); + } + +}; +/// @endcond + +} // namespace protozero + +#endif // PROTOZERO_BUFFER_VECTOR_HPP diff --git a/thirdparty/protozero/include/protozero/byteswap.hpp b/thirdparty/protozero/include/protozero/byteswap.hpp new file mode 100644 index 000000000..75cae6910 --- /dev/null +++ b/thirdparty/protozero/include/protozero/byteswap.hpp @@ -0,0 +1,108 @@ +#ifndef PROTOZERO_BYTESWAP_HPP +#define PROTOZERO_BYTESWAP_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file byteswap.hpp + * + * @brief Contains functions to swap bytes in values (for different endianness). + */ + +#include "config.hpp" + +#include <cstdint> +#include <cstring> + +namespace protozero { +namespace detail { + +inline uint32_t byteswap_impl(uint32_t value) noexcept { +#ifdef PROTOZERO_USE_BUILTIN_BSWAP + return __builtin_bswap32(value); +#else + return ((value & 0xff000000U) >> 24U) | + ((value & 0x00ff0000U) >> 8U) | + ((value & 0x0000ff00U) << 8U) | + ((value & 0x000000ffU) << 24U); +#endif +} + +inline uint64_t byteswap_impl(uint64_t value) noexcept { +#ifdef PROTOZERO_USE_BUILTIN_BSWAP + return __builtin_bswap64(value); +#else + return ((value & 0xff00000000000000ULL) >> 56U) | + ((value & 0x00ff000000000000ULL) >> 40U) | + ((value & 0x0000ff0000000000ULL) >> 24U) | + ((value & 0x000000ff00000000ULL) >> 8U) | + ((value & 0x00000000ff000000ULL) << 8U) | + ((value & 0x0000000000ff0000ULL) << 24U) | + ((value & 0x000000000000ff00ULL) << 40U) | + ((value & 0x00000000000000ffULL) << 56U); +#endif +} + +} // end namespace detail + +/// byteswap the data pointed to by ptr in-place. +inline void byteswap_inplace(uint32_t* ptr) noexcept { + *ptr = detail::byteswap_impl(*ptr); +} + +/// byteswap the data pointed to by ptr in-place. +inline void byteswap_inplace(uint64_t* ptr) noexcept { + *ptr = detail::byteswap_impl(*ptr); +} + +/// byteswap the data pointed to by ptr in-place. +inline void byteswap_inplace(int32_t* ptr) noexcept { + auto* bptr = reinterpret_cast<uint32_t*>(ptr); + *bptr = detail::byteswap_impl(*bptr); +} + +/// byteswap the data pointed to by ptr in-place. +inline void byteswap_inplace(int64_t* ptr) noexcept { + auto* bptr = reinterpret_cast<uint64_t*>(ptr); + *bptr = detail::byteswap_impl(*bptr); +} + +/// byteswap the data pointed to by ptr in-place. +inline void byteswap_inplace(float* ptr) noexcept { + static_assert(sizeof(float) == 4, "Expecting four byte float"); + + uint32_t tmp = 0; + std::memcpy(&tmp, ptr, 4); + tmp = detail::byteswap_impl(tmp); // uint32 overload + std::memcpy(ptr, &tmp, 4); +} + +/// byteswap the data pointed to by ptr in-place. +inline void byteswap_inplace(double* ptr) noexcept { + static_assert(sizeof(double) == 8, "Expecting eight byte double"); + + uint64_t tmp = 0; + std::memcpy(&tmp, ptr, 8); + tmp = detail::byteswap_impl(tmp); // uint64 overload + std::memcpy(ptr, &tmp, 8); +} + +namespace detail { + + // Added for backwards compatibility with any code that might use this + // function (even if it shouldn't have). Will be removed in a later + // version of protozero. + using ::protozero::byteswap_inplace; + +} // end namespace detail + +} // end namespace protozero + +#endif // PROTOZERO_BYTESWAP_HPP diff --git a/thirdparty/protozero/include/protozero/config.hpp b/thirdparty/protozero/include/protozero/config.hpp new file mode 100644 index 000000000..6fc774904 --- /dev/null +++ b/thirdparty/protozero/include/protozero/config.hpp @@ -0,0 +1,48 @@ +#ifndef PROTOZERO_CONFIG_HPP +#define PROTOZERO_CONFIG_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include <cassert> + +/** + * @file config.hpp + * + * @brief Contains macro checks for different configurations. + */ + +#define PROTOZERO_LITTLE_ENDIAN 1234 +#define PROTOZERO_BIG_ENDIAN 4321 + +// Find out which byte order the machine has. +#if defined(__BYTE_ORDER) +# if (__BYTE_ORDER == __LITTLE_ENDIAN) +# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN +# endif +# if (__BYTE_ORDER == __BIG_ENDIAN) +# define PROTOZERO_BYTE_ORDER PROTOZERO_BIG_ENDIAN +# endif +#else +// This probably isn't a very good default, but might do until we figure +// out something better. +# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN +#endif + +// Check whether __builtin_bswap is available +#if defined(__GNUC__) || defined(__clang__) +# define PROTOZERO_USE_BUILTIN_BSWAP +#endif + +// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +#endif // PROTOZERO_CONFIG_HPP diff --git a/thirdparty/protozero/include/protozero/data_view.hpp b/thirdparty/protozero/include/protozero/data_view.hpp new file mode 100644 index 000000000..815ceb3d2 --- /dev/null +++ b/thirdparty/protozero/include/protozero/data_view.hpp @@ -0,0 +1,236 @@ +#ifndef PROTOZERO_DATA_VIEW_HPP +#define PROTOZERO_DATA_VIEW_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file data_view.hpp + * + * @brief Contains the implementation of the data_view class. + */ + +#include "config.hpp" + +#include <algorithm> +#include <cstddef> +#include <cstring> +#include <string> +#include <utility> + +namespace protozero { + +#ifdef PROTOZERO_USE_VIEW +using data_view = PROTOZERO_USE_VIEW; +#else + +/** + * Holds a pointer to some data and a length. + * + * This class is supposed to be compatible with the std::string_view + * that will be available in C++17. + */ +class data_view { + + const char* m_data = nullptr; + std::size_t m_size = 0; + +public: + + /** + * Default constructor. Construct an empty data_view. + */ + constexpr data_view() noexcept = default; + + /** + * Create data_view from pointer and size. + * + * @param ptr Pointer to the data. + * @param length Length of the data. + */ + constexpr data_view(const char* ptr, std::size_t length) noexcept + : m_data{ptr}, + m_size{length} { + } + + /** + * Create data_view from string. + * + * @param str String with the data. + */ + data_view(const std::string& str) noexcept // NOLINT(google-explicit-constructor, hicpp-explicit-conversions) + : m_data{str.data()}, + m_size{str.size()} { + } + + /** + * Create data_view from zero-terminated string. + * + * @param ptr Pointer to the data. + */ + data_view(const char* ptr) noexcept // NOLINT(google-explicit-constructor, hicpp-explicit-conversions) + : m_data{ptr}, + m_size{std::strlen(ptr)} { + } + + /** + * Swap the contents of this object with the other. + * + * @param other Other object to swap data with. + */ + void swap(data_view& other) noexcept { + using std::swap; + swap(m_data, other.m_data); + swap(m_size, other.m_size); + } + + /// Return pointer to data. + constexpr const char* data() const noexcept { + return m_data; + } + + /// Return length of data in bytes. + constexpr std::size_t size() const noexcept { + return m_size; + } + + /// Returns true if size is 0. + constexpr bool empty() const noexcept { + return m_size == 0; + } + +#ifndef PROTOZERO_STRICT_API + /** + * Convert data view to string. + * + * @pre Must not be default constructed data_view. + * + * @deprecated to_string() is not available in C++17 string_view so it + * should not be used to make conversion to that class easier + * in the future. + */ + std::string to_string() const { + protozero_assert(m_data); + return {m_data, m_size}; + } +#endif + + /** + * Convert data view to string. + * + * @pre Must not be default constructed data_view. + */ + explicit operator std::string() const { + protozero_assert(m_data); + return {m_data, m_size}; + } + + /** + * Compares the contents of this object with the given other object. + * + * @returns 0 if they are the same, <0 if this object is smaller than + * the other or >0 if it is larger. If both objects have the + * same size returns <0 if this object is lexicographically + * before the other, >0 otherwise. + * + * @pre Must not be default constructed data_view. + */ + int compare(data_view other) const noexcept { + assert(m_data && other.m_data); + const int cmp = std::memcmp(data(), other.data(), + std::min(size(), other.size())); + if (cmp == 0) { + if (size() == other.size()) { + return 0; + } + return size() < other.size() ? -1 : 1; + } + return cmp; + } + +}; // class data_view + +/** + * Swap two data_view objects. + * + * @param lhs First object. + * @param rhs Second object. + */ +inline void swap(data_view& lhs, data_view& rhs) noexcept { + lhs.swap(rhs); +} + +/** + * Two data_view instances are equal if they have the same size and the + * same content. + * + * @param lhs First object. + * @param rhs Second object. + */ +constexpr bool operator==(const data_view lhs, const data_view rhs) noexcept { + return lhs.size() == rhs.size() && + std::equal(lhs.data(), lhs.data() + lhs.size(), rhs.data()); +} + +/** + * Two data_view instances are not equal if they have different sizes or the + * content differs. + * + * @param lhs First object. + * @param rhs Second object. + */ +constexpr bool operator!=(const data_view lhs, const data_view rhs) noexcept { + return !(lhs == rhs); +} + +/** + * Returns true if lhs.compare(rhs) < 0. + * + * @param lhs First object. + * @param rhs Second object. + */ +inline bool operator<(const data_view lhs, const data_view rhs) noexcept { + return lhs.compare(rhs) < 0; +} + +/** + * Returns true if lhs.compare(rhs) <= 0. + * + * @param lhs First object. + * @param rhs Second object. + */ +inline bool operator<=(const data_view lhs, const data_view rhs) noexcept { + return lhs.compare(rhs) <= 0; +} + +/** + * Returns true if lhs.compare(rhs) > 0. + * + * @param lhs First object. + * @param rhs Second object. + */ +inline bool operator>(const data_view lhs, const data_view rhs) noexcept { + return lhs.compare(rhs) > 0; +} + +/** + * Returns true if lhs.compare(rhs) >= 0. + * + * @param lhs First object. + * @param rhs Second object. + */ +inline bool operator>=(const data_view lhs, const data_view rhs) noexcept { + return lhs.compare(rhs) >= 0; +} + +#endif + +} // end namespace protozero + +#endif // PROTOZERO_DATA_VIEW_HPP diff --git a/thirdparty/protozero/include/protozero/exception.hpp b/thirdparty/protozero/include/protozero/exception.hpp new file mode 100644 index 000000000..a3cd0f15d --- /dev/null +++ b/thirdparty/protozero/include/protozero/exception.hpp @@ -0,0 +1,101 @@ +#ifndef PROTOZERO_EXCEPTION_HPP +#define PROTOZERO_EXCEPTION_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file exception.hpp + * + * @brief Contains the exceptions used in the protozero library. + */ + +#include <exception> + +/** + * @brief All parts of the protozero header-only library are in this namespace. + */ +namespace protozero { + +/** + * All exceptions explicitly thrown by the functions of the protozero library + * derive from this exception. + */ +struct exception : std::exception { + /// Returns the explanatory string. + const char* what() const noexcept override { + return "pbf exception"; + } +}; + +/** + * This exception is thrown when parsing a varint thats larger than allowed. + * This should never happen unless the data is corrupted. + */ +struct varint_too_long_exception : exception { + /// Returns the explanatory string. + const char* what() const noexcept override { + return "varint too long exception"; + } +}; + +/** + * This exception is thrown when the wire type of a pdf field is unknown. + * This should never happen unless the data is corrupted. + */ +struct unknown_pbf_wire_type_exception : exception { + /// Returns the explanatory string. + const char* what() const noexcept override { + return "unknown pbf field type exception"; + } +}; + +/** + * This exception is thrown when we are trying to read a field and there + * are not enough bytes left in the buffer to read it. Almost all functions + * of the pbf_reader class can throw this exception. + * + * This should never happen unless the data is corrupted or you have + * initialized the pbf_reader object with incomplete data. + */ +struct end_of_buffer_exception : exception { + /// Returns the explanatory string. + const char* what() const noexcept override { + return "end of buffer exception"; + } +}; + +/** + * This exception is thrown when a tag has an invalid value. Tags must be + * unsigned integers between 1 and 2^29-1. Tags between 19000 and 19999 are + * not allowed. See + * https://developers.google.com/protocol-buffers/docs/proto#assigning-tags + */ +struct invalid_tag_exception : exception { + /// Returns the explanatory string. + const char* what() const noexcept override { + return "invalid tag exception"; + } +}; + +/** + * This exception is thrown when a length field of a packed repeated field is + * invalid. For fixed size types the length must be a multiple of the size of + * the type. + */ +struct invalid_length_exception : exception { + /// Returns the explanatory string. + const char* what() const noexcept override { + return "invalid length exception"; + } +}; + +} // end namespace protozero + +#endif // PROTOZERO_EXCEPTION_HPP diff --git a/thirdparty/protozero/include/protozero/iterators.hpp b/thirdparty/protozero/include/protozero/iterators.hpp new file mode 100644 index 000000000..ee8ef8ecf --- /dev/null +++ b/thirdparty/protozero/include/protozero/iterators.hpp @@ -0,0 +1,481 @@ +#ifndef PROTOZERO_ITERATORS_HPP +#define PROTOZERO_ITERATORS_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file iterators.hpp + * + * @brief Contains the iterators for access to packed repeated fields. + */ + +#include "config.hpp" +#include "varint.hpp" + +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN +# include <protozero/byteswap.hpp> +#endif + +#include <algorithm> +#include <cstring> +#include <iterator> +#include <utility> + +namespace protozero { + +/** + * A range of iterators based on std::pair. Created from beginning and + * end iterators. Used as a return type from some pbf_reader methods + * that is easy to use with range-based for loops. + */ +template <typename T, typename P = std::pair<T, T>> +class iterator_range : +#ifdef PROTOZERO_STRICT_API + protected +#else + public +#endif + P { + +public: + + /// The type of the iterators in this range. + using iterator = T; + + /// The value type of the underlying iterator. + using value_type = typename std::iterator_traits<T>::value_type; + + /** + * Default constructor. Create empty iterator_range. + */ + constexpr iterator_range() : + P{iterator{}, iterator{}} { + } + + /** + * Create iterator range from two iterators. + * + * @param first_iterator Iterator to beginning of range. + * @param last_iterator Iterator to end of range. + */ + constexpr iterator_range(iterator&& first_iterator, iterator&& last_iterator) : + P{std::forward<iterator>(first_iterator), + std::forward<iterator>(last_iterator)} { + } + + /// Return iterator to beginning of range. + constexpr iterator begin() const noexcept { + return this->first; + } + + /// Return iterator to end of range. + constexpr iterator end() const noexcept { + return this->second; + } + + /// Return iterator to beginning of range. + constexpr iterator cbegin() const noexcept { + return this->first; + } + + /// Return iterator to end of range. + constexpr iterator cend() const noexcept { + return this->second; + } + + /** + * Return true if this range is empty. + * + * Complexity: Constant. + */ + constexpr bool empty() const noexcept { + return begin() == end(); + } + + /** + * Get the size of the range, ie the number of elements it contains. + * + * Complexity: Constant or linear depending on the underlaying iterator. + */ + std::size_t size() const noexcept { + return static_cast<size_t>(std::distance(begin(), end())); + } + + /** + * Get element at the beginning of the range. + * + * @pre Range must not be empty. + */ + value_type front() const { + protozero_assert(!empty()); + return *(this->first); + } + + /** + * Advance beginning of range by one. + * + * @pre Range must not be empty. + */ + void drop_front() { + protozero_assert(!empty()); + ++this->first; + } + + /** + * Swap the contents of this range with the other. + * + * @param other Other range to swap data with. + */ + void swap(iterator_range& other) noexcept { + using std::swap; + swap(this->first, other.first); + swap(this->second, other.second); + } + +}; // struct iterator_range + +/** + * Swap two iterator_ranges. + * + * @param lhs First range. + * @param rhs Second range. + */ +template <typename T> +inline void swap(iterator_range<T>& lhs, iterator_range<T>& rhs) noexcept { + lhs.swap(rhs); +} + +/** + * A forward iterator used for accessing packed repeated fields of fixed + * length (fixed32, sfixed32, float, double). + */ +template <typename T> +class const_fixed_iterator { + + /// Pointer to current iterator position + const char* m_data = nullptr; + +public: + + /// @cond usual iterator functions not documented + + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + const_fixed_iterator() noexcept = default; + + explicit const_fixed_iterator(const char* data) noexcept : + m_data{data} { + } + + const_fixed_iterator(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator(const_fixed_iterator&&) noexcept = default; + + const_fixed_iterator& operator=(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator& operator=(const_fixed_iterator&&) noexcept = default; + + ~const_fixed_iterator() noexcept = default; + + value_type operator*() const noexcept { + value_type result; + std::memcpy(&result, m_data, sizeof(value_type)); +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN + byteswap_inplace(&result); +#endif + return result; + } + + const_fixed_iterator& operator++() noexcept { + m_data += sizeof(value_type); + return *this; + } + + const_fixed_iterator operator++(int) noexcept { + const const_fixed_iterator tmp{*this}; + ++(*this); + return tmp; + } + + const_fixed_iterator& operator--() noexcept { + m_data -= sizeof(value_type); + return *this; + } + + const_fixed_iterator operator--(int) noexcept { + const const_fixed_iterator tmp{*this}; + --(*this); + return tmp; + } + + friend bool operator==(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept { + return lhs.m_data == rhs.m_data; + } + + friend bool operator!=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept { + return !(lhs == rhs); + } + + friend bool operator<(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept { + return lhs.m_data < rhs.m_data; + } + + friend bool operator>(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept { + return rhs < lhs; + } + + friend bool operator<=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept { + return !(lhs > rhs); + } + + friend bool operator>=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept { + return !(lhs < rhs); + } + + const_fixed_iterator& operator+=(difference_type val) noexcept { + m_data += (sizeof(value_type) * val); + return *this; + } + + friend const_fixed_iterator operator+(const_fixed_iterator lhs, difference_type rhs) noexcept { + const_fixed_iterator tmp{lhs}; + tmp.m_data += (sizeof(value_type) * rhs); + return tmp; + } + + friend const_fixed_iterator operator+(difference_type lhs, const_fixed_iterator rhs) noexcept { + const_fixed_iterator tmp{rhs}; + tmp.m_data += (sizeof(value_type) * lhs); + return tmp; + } + + const_fixed_iterator& operator-=(difference_type val) noexcept { + m_data -= (sizeof(value_type) * val); + return *this; + } + + friend const_fixed_iterator operator-(const_fixed_iterator lhs, difference_type rhs) noexcept { + const_fixed_iterator tmp{lhs}; + tmp.m_data -= (sizeof(value_type) * rhs); + return tmp; + } + + friend difference_type operator-(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept { + return static_cast<difference_type>(lhs.m_data - rhs.m_data) / static_cast<difference_type>(sizeof(T)); + } + + value_type operator[](difference_type n) const noexcept { + return *(*this + n); + } + + /// @endcond + +}; // class const_fixed_iterator + +/** + * A forward iterator used for accessing packed repeated varint fields + * (int32, uint32, int64, uint64, bool, enum). + */ +template <typename T> +class const_varint_iterator { + +protected: + + /// Pointer to current iterator position + const char* m_data = nullptr; // NOLINT(misc-non-private-member-variables-in-classes, cppcoreguidelines-non-private-member-variables-in-classes,-warnings-as-errors) + + /// Pointer to end iterator position + const char* m_end = nullptr; // NOLINT(misc-non-private-member-variables-in-classes, cppcoreguidelines-non-private-member-variables-in-classes,-warnings-as-errors) + +public: + + /// @cond usual iterator functions not documented + + using iterator_category = std::forward_iterator_tag; + using value_type = T; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + static difference_type distance(const_varint_iterator begin, const_varint_iterator end) noexcept { + // The "distance" between default initialized const_varint_iterator's + // is always 0. + if (!begin.m_data) { + return 0; + } + // We know that each varint contains exactly one byte with the most + // significant bit not set. We can use this to quickly figure out + // how many varints there are without actually decoding the varints. + return std::count_if(begin.m_data, end.m_data, [](char c) noexcept { + return (static_cast<unsigned char>(c) & 0x80U) == 0; + }); + } + + const_varint_iterator() noexcept = default; + + const_varint_iterator(const char* data, const char* end) noexcept : + m_data{data}, + m_end{end} { + } + + const_varint_iterator(const const_varint_iterator&) noexcept = default; + const_varint_iterator(const_varint_iterator&&) noexcept = default; + + const_varint_iterator& operator=(const const_varint_iterator&) noexcept = default; + const_varint_iterator& operator=(const_varint_iterator&&) noexcept = default; + + ~const_varint_iterator() noexcept = default; + + value_type operator*() const { + protozero_assert(m_data); + const char* d = m_data; // will be thrown away + return static_cast<value_type>(decode_varint(&d, m_end)); + } + + const_varint_iterator& operator++() { + protozero_assert(m_data); + skip_varint(&m_data, m_end); + return *this; + } + + const_varint_iterator operator++(int) { + protozero_assert(m_data); + const const_varint_iterator tmp{*this}; + ++(*this); + return tmp; + } + + bool operator==(const const_varint_iterator& rhs) const noexcept { + return m_data == rhs.m_data && m_end == rhs.m_end; + } + + bool operator!=(const const_varint_iterator& rhs) const noexcept { + return !(*this == rhs); + } + + /// @endcond + +}; // class const_varint_iterator + +/** + * A forward iterator used for accessing packed repeated svarint fields + * (sint32, sint64). + */ +template <typename T> +class const_svarint_iterator : public const_varint_iterator<T> { + +public: + + /// @cond usual iterator functions not documented + + using iterator_category = std::forward_iterator_tag; + using value_type = T; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + const_svarint_iterator() noexcept : + const_varint_iterator<T>{} { + } + + const_svarint_iterator(const char* data, const char* end) noexcept : + const_varint_iterator<T>{data, end} { + } + + const_svarint_iterator(const const_svarint_iterator&) = default; + const_svarint_iterator(const_svarint_iterator&&) noexcept = default; + + const_svarint_iterator& operator=(const const_svarint_iterator&) = default; + const_svarint_iterator& operator=(const_svarint_iterator&&) noexcept = default; + + ~const_svarint_iterator() = default; + + value_type operator*() const { + protozero_assert(this->m_data); + const char* d = this->m_data; // will be thrown away + return static_cast<value_type>(decode_zigzag64(decode_varint(&d, this->m_end))); + } + + const_svarint_iterator& operator++() { + protozero_assert(this->m_data); + skip_varint(&this->m_data, this->m_end); + return *this; + } + + const_svarint_iterator operator++(int) { + protozero_assert(this->m_data); + const const_svarint_iterator tmp{*this}; + ++(*this); + return tmp; + } + + /// @endcond + +}; // class const_svarint_iterator + +} // end namespace protozero + +namespace std { + + // Specialize std::distance for all the protozero iterators. Because + // functions can't be partially specialized, we have to do this for + // every value_type we are using. + + /// @cond individual overloads do not need to be documented + + template <> + inline typename protozero::const_varint_iterator<int32_t>::difference_type + distance<protozero::const_varint_iterator<int32_t>>(protozero::const_varint_iterator<int32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name) + protozero::const_varint_iterator<int32_t> last) { + return protozero::const_varint_iterator<int32_t>::distance(first, last); + } + + template <> + inline typename protozero::const_varint_iterator<int64_t>::difference_type + distance<protozero::const_varint_iterator<int64_t>>(protozero::const_varint_iterator<int64_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name) + protozero::const_varint_iterator<int64_t> last) { + return protozero::const_varint_iterator<int64_t>::distance(first, last); + } + + template <> + inline typename protozero::const_varint_iterator<uint32_t>::difference_type + distance<protozero::const_varint_iterator<uint32_t>>(protozero::const_varint_iterator<uint32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name) + protozero::const_varint_iterator<uint32_t> last) { + return protozero::const_varint_iterator<uint32_t>::distance(first, last); + } + + template <> + inline typename protozero::const_varint_iterator<uint64_t>::difference_type + distance<protozero::const_varint_iterator<uint64_t>>(protozero::const_varint_iterator<uint64_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name) + protozero::const_varint_iterator<uint64_t> last) { + return protozero::const_varint_iterator<uint64_t>::distance(first, last); + } + + template <> + inline typename protozero::const_svarint_iterator<int32_t>::difference_type + distance<protozero::const_svarint_iterator<int32_t>>(protozero::const_svarint_iterator<int32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name) + protozero::const_svarint_iterator<int32_t> last) { + return protozero::const_svarint_iterator<int32_t>::distance(first, last); + } + + template <> + inline typename protozero::const_svarint_iterator<int64_t>::difference_type + distance<protozero::const_svarint_iterator<int64_t>>(protozero::const_svarint_iterator<int64_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name) + protozero::const_svarint_iterator<int64_t> last) { + return protozero::const_svarint_iterator<int64_t>::distance(first, last); + } + + /// @endcond + +} // end namespace std + +#endif // PROTOZERO_ITERATORS_HPP diff --git a/thirdparty/protozero/include/protozero/pbf_builder.hpp b/thirdparty/protozero/include/protozero/pbf_builder.hpp new file mode 100644 index 000000000..71a2dec2b --- /dev/null +++ b/thirdparty/protozero/include/protozero/pbf_builder.hpp @@ -0,0 +1,32 @@ +#ifndef PROTOZERO_PBF_BUILDER_HPP +#define PROTOZERO_PBF_BUILDER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_builder.hpp + * + * @brief Contains the pbf_builder template class. + */ + +#include "basic_pbf_builder.hpp" +#include "pbf_writer.hpp" + +#include <string> + +namespace protozero { + +/// Specialization of basic_pbf_builder using std::string as buffer type. +template <typename T> +using pbf_builder = basic_pbf_builder<std::string, T>; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_BUILDER_HPP diff --git a/thirdparty/protozero/include/protozero/pbf_message.hpp b/thirdparty/protozero/include/protozero/pbf_message.hpp new file mode 100644 index 000000000..393078b9b --- /dev/null +++ b/thirdparty/protozero/include/protozero/pbf_message.hpp @@ -0,0 +1,184 @@ +#ifndef PROTOZERO_PBF_MESSAGE_HPP +#define PROTOZERO_PBF_MESSAGE_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_message.hpp + * + * @brief Contains the pbf_message template class. + */ + +#include "pbf_reader.hpp" +#include "types.hpp" + +#include <type_traits> + +namespace protozero { + +/** + * This class represents a protobuf message. Either a top-level message or + * a nested sub-message. Top-level messages can be created from any buffer + * with a pointer and length: + * + * @code + * enum class Message : protozero::pbf_tag_type { + * ... + * }; + * + * std::string buffer; + * // fill buffer... + * pbf_message<Message> message{buffer.data(), buffer.size()}; + * @endcode + * + * Sub-messages are created using get_message(): + * + * @code + * enum class SubMessage : protozero::pbf_tag_type { + * ... + * }; + * + * pbf_message<Message> message{...}; + * message.next(); + * pbf_message<SubMessage> submessage = message.get_message(); + * @endcode + * + * All methods of the pbf_message class except get_bytes() and get_string() + * provide the strong exception guarantee, ie they either succeed or do not + * change the pbf_message object they are called on. Use the get_data() method + * instead of get_bytes() or get_string(), if you need this guarantee. + * + * This template class is based on the pbf_reader class and has all the same + * methods. The difference is that whereever the pbf_reader class takes an + * integer tag, this template class takes a tag of the template type T. + * + * Read the tutorial to understand how this class is used. + */ +template <typename T> +class pbf_message : public pbf_reader { + + static_assert(std::is_same<pbf_tag_type, std::underlying_type_t<T>>::value, + "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + /// The type of messages this class will read. + using enum_type = T; + + /** + * Construct a pbf_message. All arguments are forwarded to the pbf_reader + * parent class. + */ + template <typename... Args> + pbf_message(Args&&... args) noexcept : // NOLINT(google-explicit-constructor, hicpp-explicit-conversions) + pbf_reader{std::forward<Args>(args)...} { + } + + /** + * Set next field in the message as the current field. This is usually + * called in a while loop: + * + * @code + * pbf_message<...> message(...); + * while (message.next()) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field, `false` if not. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now. + */ + bool next() { + return pbf_reader::next(); + } + + /** + * Set next field with given tag in the message as the current field. + * Fields with other tags are skipped. This is usually called in a while + * loop for repeated fields: + * + * @code + * pbf_message<Example1> message{...}; + * while (message.next(Example1::repeated_fixed64_r)) { + * // handle field + * } + * @endcode + * + * or you can call it just once to get the one field with this tag: + * + * @code + * pbf_message<Example1> message{...}; + * if (message.next(Example1::required_uint32_x)) { + * // handle field + * } + * @endcode + * + * Note that this will not check the wire type. The two-argument version + * of this function will also check the wire type. + * + * @returns `true` if there is a next field with this tag. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now with the given tag. + */ + bool next(T next_tag) { + return pbf_reader::next(pbf_tag_type(next_tag)); + } + + /** + * Set next field with given tag and wire type in the message as the + * current field. Fields with other tags are skipped. This is usually + * called in a while loop for repeated fields: + * + * @code + * pbf_message<Example1> message{...}; + * while (message.next(Example1::repeated_fixed64_r, pbf_wire_type::varint)) { + * // handle field + * } + * @endcode + * + * or you can call it just once to get the one field with this tag: + * + * @code + * pbf_message<Example1> message{...}; + * if (message.next(Example1::required_uint32_x, pbf_wire_type::varint)) { + * // handle field + * } + * @endcode + * + * Note that this will also check the wire type. The one-argument version + * of this function will not check the wire type. + * + * @returns `true` if there is a next field with this tag. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now with the given tag. + */ + bool next(T next_tag, pbf_wire_type type) { + return pbf_reader::next(pbf_tag_type(next_tag), type); + } + + /** + * The tag of the current field. The tag is the enum value for the field + * number from the description in the .proto file. + * + * Call next() before calling this function to set the current field. + * + * @returns tag of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + T tag() const noexcept { + return T(pbf_reader::tag()); + } + +}; // class pbf_message + +} // end namespace protozero + +#endif // PROTOZERO_PBF_MESSAGE_HPP diff --git a/thirdparty/protozero/include/protozero/pbf_reader.hpp b/thirdparty/protozero/include/protozero/pbf_reader.hpp new file mode 100644 index 000000000..fab4dceb7 --- /dev/null +++ b/thirdparty/protozero/include/protozero/pbf_reader.hpp @@ -0,0 +1,977 @@ +#ifndef PROTOZERO_PBF_READER_HPP +#define PROTOZERO_PBF_READER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_reader.hpp + * + * @brief Contains the pbf_reader class. + */ + +#include "config.hpp" +#include "data_view.hpp" +#include "exception.hpp" +#include "iterators.hpp" +#include "types.hpp" +#include "varint.hpp" + +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN +# include <protozero/byteswap.hpp> +#endif + +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <string> +#include <utility> + +namespace protozero { + +/** + * This class represents a protobuf message. Either a top-level message or + * a nested sub-message. Top-level messages can be created from any buffer + * with a pointer and length: + * + * @code + * std::string buffer; + * // fill buffer... + * pbf_reader message{buffer.data(), buffer.size()}; + * @endcode + * + * Sub-messages are created using get_message(): + * + * @code + * pbf_reader message{...}; + * message.next(); + * pbf_reader submessage = message.get_message(); + * @endcode + * + * All methods of the pbf_reader class except get_bytes() and get_string() + * provide the strong exception guarantee, ie they either succeed or do not + * change the pbf_reader object they are called on. Use the get_view() method + * instead of get_bytes() or get_string(), if you need this guarantee. + */ +class pbf_reader { + + // A pointer to the next unread data. + const char* m_data = nullptr; + + // A pointer to one past the end of data. + const char* m_end = nullptr; + + // The wire type of the current field. + pbf_wire_type m_wire_type = pbf_wire_type::unknown; + + // The tag of the current field. + pbf_tag_type m_tag = 0; + + template <typename T> + T get_fixed() { + T result; + const char* tmp_data = m_data; + skip_bytes(sizeof(T)); + std::memcpy(&result, tmp_data, sizeof(T)); +#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN + byteswap_inplace(&result); +#endif + return result; + } + + template <typename T> + iterator_range<const_fixed_iterator<T>> packed_fixed() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + const auto len = get_len_and_skip(); + if (len % sizeof(T) != 0) { + throw invalid_length_exception{}; + } + return {const_fixed_iterator<T>(m_data - len), + const_fixed_iterator<T>(m_data)}; + } + + template <typename T> + T get_varint() { + const auto val = static_cast<T>(decode_varint(&m_data, m_end)); + return val; + } + + template <typename T> + T get_svarint() { + protozero_assert((has_wire_type(pbf_wire_type::varint) || has_wire_type(pbf_wire_type::length_delimited)) && "not a varint"); + return static_cast<T>(decode_zigzag64(decode_varint(&m_data, m_end))); + } + + pbf_length_type get_length() { + return get_varint<pbf_length_type>(); + } + + void skip_bytes(pbf_length_type len) { + if (m_end - m_data < static_cast<ptrdiff_t>(len)) { + throw end_of_buffer_exception{}; + } + m_data += len; + +#ifndef NDEBUG + // In debug builds reset the tag to zero so that we can detect (some) + // wrong code. + m_tag = 0; +#endif + } + + pbf_length_type get_len_and_skip() { + const auto len = get_length(); + skip_bytes(len); + return len; + } + + template <typename T> + iterator_range<T> get_packed() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + const auto len = get_len_and_skip(); + return {T{m_data - len, m_data}, + T{m_data, m_data}}; + } + +public: + + /** + * Construct a pbf_reader message from a data_view. The pointer from the + * data_view will be stored inside the pbf_reader object, no data is + * copied. So you must make sure the view stays valid as long as the + * pbf_reader object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + explicit pbf_reader(const data_view& view) noexcept + : m_data{view.data()}, + m_end{view.data() + view.size()} { + } + + /** + * Construct a pbf_reader message from a data pointer and a length. The + * pointer will be stored inside the pbf_reader object, no data is copied. + * So you must make sure the buffer stays valid as long as the pbf_reader + * object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + pbf_reader(const char* data, std::size_t size) noexcept + : m_data{data}, + m_end{data + size} { + } + +#ifndef PROTOZERO_STRICT_API + /** + * Construct a pbf_reader message from a data pointer and a length. The + * pointer will be stored inside the pbf_reader object, no data is copied. + * So you must make sure the buffer stays valid as long as the pbf_reader + * object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + * @deprecated Use one of the other constructors. + */ + explicit pbf_reader(const std::pair<const char*, std::size_t>& data) noexcept + : m_data{data.first}, + m_end{data.first + data.second} { + } +#endif + + /** + * Construct a pbf_reader message from a std::string. A pointer to the + * string internals will be stored inside the pbf_reader object, no data + * is copied. So you must make sure the string is unchanged as long as the + * pbf_reader object is used. + * + * The string must contain a complete protobuf message. + * + * @post There is no current field. + */ + explicit pbf_reader(const std::string& data) noexcept + : m_data{data.data()}, + m_end{data.data() + data.size()} { + } + + /** + * pbf_reader can be default constructed and behaves like it has an empty + * buffer. + */ + pbf_reader() noexcept = default; + + /// pbf_reader messages can be copied trivially. + pbf_reader(const pbf_reader&) noexcept = default; + + /// pbf_reader messages can be moved trivially. + pbf_reader(pbf_reader&&) noexcept = default; + + /// pbf_reader messages can be copied trivially. + pbf_reader& operator=(const pbf_reader& other) noexcept = default; + + /// pbf_reader messages can be moved trivially. + pbf_reader& operator=(pbf_reader&& other) noexcept = default; + + ~pbf_reader() = default; + + /** + * Swap the contents of this object with the other. + * + * @param other Other object to swap data with. + */ + void swap(pbf_reader& other) noexcept { + using std::swap; + swap(m_data, other.m_data); + swap(m_end, other.m_end); + swap(m_wire_type, other.m_wire_type); + swap(m_tag, other.m_tag); + } + + /** + * In a boolean context the pbf_reader class evaluates to `true` if there + * are still fields available and to `false` if the last field has been + * read. + */ + operator bool() const noexcept { // NOLINT(google-explicit-constructor, hicpp-explicit-conversions) + return m_data != m_end; + } + + /** + * Get a view of the not yet read data. + */ + data_view data() const noexcept { + return {m_data, static_cast<std::size_t>(m_end - m_data)}; + } + + /** + * Return the length in bytes of the current message. If you have + * already called next() and/or any of the get_*() functions, this will + * return the remaining length. + * + * This can, for instance, be used to estimate the space needed for a + * buffer. Of course you have to know reasonably well what data to expect + * and how it is encoded for this number to have any meaning. + */ + std::size_t length() const noexcept { + return static_cast<std::size_t>(m_end - m_data); + } + + /** + * Set next field in the message as the current field. This is usually + * called in a while loop: + * + * @code + * pbf_reader message(...); + * while (message.next()) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field, `false` if not. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now. + */ + bool next() { + if (m_data == m_end) { + return false; + } + + const auto value = get_varint<uint32_t>(); + m_tag = static_cast<pbf_tag_type>(value >> 3U); + + // tags 0 and 19000 to 19999 are not allowed as per + // https://developers.google.com/protocol-buffers/docs/proto#assigning-tags + if (m_tag == 0 || (m_tag >= 19000 && m_tag <= 19999)) { + throw invalid_tag_exception{}; + } + + m_wire_type = pbf_wire_type(value & 0x07U); + switch (m_wire_type) { + case pbf_wire_type::varint: + case pbf_wire_type::fixed64: + case pbf_wire_type::length_delimited: + case pbf_wire_type::fixed32: + break; + default: + throw unknown_pbf_wire_type_exception{}; + } + + return true; + } + + /** + * Set next field with given tag in the message as the current field. + * Fields with other tags are skipped. This is usually called in a while + * loop for repeated fields: + * + * @code + * pbf_reader message{...}; + * while (message.next(17)) { + * // handle field + * } + * @endcode + * + * or you can call it just once to get the one field with this tag: + * + * @code + * pbf_reader message{...}; + * if (message.next(17)) { + * // handle field + * } + * @endcode + * + * Note that this will not check the wire type. The two-argument version + * of this function will also check the wire type. + * + * @returns `true` if there is a next field with this tag. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now with the given tag. + */ + bool next(pbf_tag_type next_tag) { + while (next()) { + if (m_tag == next_tag) { + return true; + } + skip(); + } + return false; + } + + /** + * Set next field with given tag and wire type in the message as the + * current field. Fields with other tags are skipped. This is usually + * called in a while loop for repeated fields: + * + * @code + * pbf_reader message{...}; + * while (message.next(17, pbf_wire_type::varint)) { + * // handle field + * } + * @endcode + * + * or you can call it just once to get the one field with this tag: + * + * @code + * pbf_reader message{...}; + * if (message.next(17, pbf_wire_type::varint)) { + * // handle field + * } + * @endcode + * + * Note that this will also check the wire type. The one-argument version + * of this function will not check the wire type. + * + * @returns `true` if there is a next field with this tag. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now with the given tag. + */ + bool next(pbf_tag_type next_tag, pbf_wire_type type) { + while (next()) { + if (m_tag == next_tag && m_wire_type == type) { + return true; + } + skip(); + } + return false; + } + + /** + * The tag of the current field. The tag is the field number from the + * description in the .proto file. + * + * Call next() before calling this function to set the current field. + * + * @returns tag of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + pbf_tag_type tag() const noexcept { + return m_tag; + } + + /** + * Get the wire type of the current field. The wire types are: + * + * * 0 - varint + * * 1 - 64 bit + * * 2 - length-delimited + * * 5 - 32 bit + * + * All other types are illegal. + * + * Call next() before calling this function to set the current field. + * + * @returns wire type of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + pbf_wire_type wire_type() const noexcept { + return m_wire_type; + } + + /** + * Get the tag and wire type of the current field in one integer suitable + * for comparison with a switch statement. + * + * Use it like this: + * + * @code + * pbf_reader message{...}; + * while (message.next()) { + * switch (message.tag_and_type()) { + * case tag_and_type(17, pbf_wire_type::length_delimited): + * .... + * break; + * case tag_and_type(21, pbf_wire_type::varint): + * .... + * break; + * default: + * message.skip(); + * } + * } + * @endcode + */ + uint32_t tag_and_type() const noexcept { + return protozero::tag_and_type(tag(), wire_type()); + } + + /** + * Check the wire type of the current field. + * + * @returns `true` if the current field has the given wire type. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + bool has_wire_type(pbf_wire_type type) const noexcept { + return wire_type() == type; + } + + /** + * Consume the current field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @post The current field was consumed and there is no current field now. + */ + void skip() { + protozero_assert(tag() != 0 && "call next() before calling skip()"); + switch (wire_type()) { + case pbf_wire_type::varint: + skip_varint(&m_data, m_end); + break; + case pbf_wire_type::fixed64: + skip_bytes(8); + break; + case pbf_wire_type::length_delimited: + skip_bytes(get_length()); + break; + case pbf_wire_type::fixed32: + skip_bytes(4); + break; + default: + break; + } + } + + ///@{ + /** + * @name Scalar field accessor functions + */ + + /** + * Consume and return value of current "bool" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bool". + * @post The current field was consumed and there is no current field now. + */ + bool get_bool() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + const char* value = m_data; + skip_varint(&m_data, m_end); + return *value != 0; + } + + /** + * Consume and return value of current "enum" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "enum". + * @post The current field was consumed and there is no current field now. + */ + int32_t get_enum() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint<int32_t>(); + } + + /** + * Consume and return value of current "int32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int32". + * @post The current field was consumed and there is no current field now. + */ + int32_t get_int32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint<int32_t>(); + } + + /** + * Consume and return value of current "sint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint32". + * @post The current field was consumed and there is no current field now. + */ + int32_t get_sint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint<int32_t>(); + } + + /** + * Consume and return value of current "uint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint32". + * @post The current field was consumed and there is no current field now. + */ + uint32_t get_uint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint<uint32_t>(); + } + + /** + * Consume and return value of current "int64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int64". + * @post The current field was consumed and there is no current field now. + */ + int64_t get_int64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint<int64_t>(); + } + + /** + * Consume and return value of current "sint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint64". + * @post The current field was consumed and there is no current field now. + */ + int64_t get_sint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint<int64_t>(); + } + + /** + * Consume and return value of current "uint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint64". + * @post The current field was consumed and there is no current field now. + */ + uint64_t get_uint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint<uint64_t>(); + } + + /** + * Consume and return value of current "fixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed32". + * @post The current field was consumed and there is no current field now. + */ + uint32_t get_fixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed<uint32_t>(); + } + + /** + * Consume and return value of current "sfixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed32". + * @post The current field was consumed and there is no current field now. + */ + int32_t get_sfixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed<int32_t>(); + } + + /** + * Consume and return value of current "fixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed64". + * @post The current field was consumed and there is no current field now. + */ + uint64_t get_fixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed<uint64_t>(); + } + + /** + * Consume and return value of current "sfixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed64". + * @post The current field was consumed and there is no current field now. + */ + int64_t get_sfixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed<int64_t>(); + } + + /** + * Consume and return value of current "float" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "float". + * @post The current field was consumed and there is no current field now. + */ + float get_float() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed<float>(); + } + + /** + * Consume and return value of current "double" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "double". + * @post The current field was consumed and there is no current field now. + */ + double get_double() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed<double>(); + } + + /** + * Consume and return value of current "bytes", "string", or "message" + * field. + * + * @returns A data_view object. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes", "string", or "message". + * @post The current field was consumed and there is no current field now. + */ + data_view get_view() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message"); + const auto len = get_len_and_skip(); + return {m_data - len, len}; + } + +#ifndef PROTOZERO_STRICT_API + /** + * Consume and return value of current "bytes" or "string" field. + * + * @returns A pair with a pointer to the data and the length of the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes" or "string". + * @post The current field was consumed and there is no current field now. + */ + std::pair<const char*, pbf_length_type> get_data() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message"); + const auto len = get_len_and_skip(); + return {m_data - len, len}; + } +#endif + + /** + * Consume and return value of current "bytes" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes". + * @post The current field was consumed and there is no current field now. + */ + std::string get_bytes() { + return std::string(get_view()); + } + + /** + * Consume and return value of current "string" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "string". + * @post The current field was consumed and there is no current field now. + */ + std::string get_string() { + return std::string(get_view()); + } + + /** + * Consume and return value of current "message" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "message". + * @post The current field was consumed and there is no current field now. + */ + pbf_reader get_message() { + return pbf_reader{get_view()}; + } + + ///@} + + /// Forward iterator for iterating over bool (int32 varint) values. + using const_bool_iterator = const_varint_iterator< int32_t>; + + /// Forward iterator for iterating over enum (int32 varint) values. + using const_enum_iterator = const_varint_iterator< int32_t>; + + /// Forward iterator for iterating over int32 (varint) values. + using const_int32_iterator = const_varint_iterator< int32_t>; + + /// Forward iterator for iterating over sint32 (varint) values. + using const_sint32_iterator = const_svarint_iterator<int32_t>; + + /// Forward iterator for iterating over uint32 (varint) values. + using const_uint32_iterator = const_varint_iterator<uint32_t>; + + /// Forward iterator for iterating over int64 (varint) values. + using const_int64_iterator = const_varint_iterator< int64_t>; + + /// Forward iterator for iterating over sint64 (varint) values. + using const_sint64_iterator = const_svarint_iterator<int64_t>; + + /// Forward iterator for iterating over uint64 (varint) values. + using const_uint64_iterator = const_varint_iterator<uint64_t>; + + /// Forward iterator for iterating over fixed32 values. + using const_fixed32_iterator = const_fixed_iterator<uint32_t>; + + /// Forward iterator for iterating over sfixed32 values. + using const_sfixed32_iterator = const_fixed_iterator<int32_t>; + + /// Forward iterator for iterating over fixed64 values. + using const_fixed64_iterator = const_fixed_iterator<uint64_t>; + + /// Forward iterator for iterating over sfixed64 values. + using const_sfixed64_iterator = const_fixed_iterator<int64_t>; + + /// Forward iterator for iterating over float values. + using const_float_iterator = const_fixed_iterator<float>; + + /// Forward iterator for iterating over double values. + using const_double_iterator = const_fixed_iterator<double>; + + ///@{ + /** + * @name Repeated packed field accessor functions + */ + + /** + * Consume current "repeated packed bool" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed bool". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_bool_iterator> get_packed_bool() { + return get_packed<pbf_reader::const_bool_iterator>(); + } + + /** + * Consume current "repeated packed enum" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed enum". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_enum_iterator> get_packed_enum() { + return get_packed<pbf_reader::const_enum_iterator>(); + } + + /** + * Consume current "repeated packed int32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int32". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_int32_iterator> get_packed_int32() { + return get_packed<pbf_reader::const_int32_iterator>(); + } + + /** + * Consume current "repeated packed sint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint32". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_sint32_iterator> get_packed_sint32() { + return get_packed<pbf_reader::const_sint32_iterator>(); + } + + /** + * Consume current "repeated packed uint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint32". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_uint32_iterator> get_packed_uint32() { + return get_packed<pbf_reader::const_uint32_iterator>(); + } + + /** + * Consume current "repeated packed int64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int64". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_int64_iterator> get_packed_int64() { + return get_packed<pbf_reader::const_int64_iterator>(); + } + + /** + * Consume current "repeated packed sint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint64". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_sint64_iterator> get_packed_sint64() { + return get_packed<pbf_reader::const_sint64_iterator>(); + } + + /** + * Consume current "repeated packed uint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint64". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_uint64_iterator> get_packed_uint64() { + return get_packed<pbf_reader::const_uint64_iterator>(); + } + + /** + * Consume current "repeated packed fixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed32". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_fixed32_iterator> get_packed_fixed32() { + return packed_fixed<uint32_t>(); + } + + /** + * Consume current "repeated packed sfixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed32". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_sfixed32_iterator> get_packed_sfixed32() { + return packed_fixed<int32_t>(); + } + + /** + * Consume current "repeated packed fixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed64". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_fixed64_iterator> get_packed_fixed64() { + return packed_fixed<uint64_t>(); + } + + /** + * Consume current "repeated packed sfixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed64". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_sfixed64_iterator> get_packed_sfixed64() { + return packed_fixed<int64_t>(); + } + + /** + * Consume current "repeated packed float" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed float". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_float_iterator> get_packed_float() { + return packed_fixed<float>(); + } + + /** + * Consume current "repeated packed double" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed double". + * @post The current field was consumed and there is no current field now. + */ + iterator_range<pbf_reader::const_double_iterator> get_packed_double() { + return packed_fixed<double>(); + } + + ///@} + +}; // class pbf_reader + +/** + * Swap two pbf_reader objects. + * + * @param lhs First object. + * @param rhs Second object. + */ +inline void swap(pbf_reader& lhs, pbf_reader& rhs) noexcept { + lhs.swap(rhs); +} + +} // end namespace protozero + +#endif // PROTOZERO_PBF_READER_HPP diff --git a/thirdparty/protozero/include/protozero/pbf_writer.hpp b/thirdparty/protozero/include/protozero/pbf_writer.hpp new file mode 100644 index 000000000..9a07bd5b9 --- /dev/null +++ b/thirdparty/protozero/include/protozero/pbf_writer.hpp @@ -0,0 +1,76 @@ +#ifndef PROTOZERO_PBF_WRITER_HPP +#define PROTOZERO_PBF_WRITER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_writer.hpp + * + * @brief Contains the pbf_writer class. + */ + +#include "basic_pbf_writer.hpp" +#include "buffer_string.hpp" + +#include <cstdint> +#include <string> + +namespace protozero { + +/** + * Specialization of basic_pbf_writer using std::string as buffer type. + */ +using pbf_writer = basic_pbf_writer<std::string>; + +/// Class for generating packed repeated bool fields. +using packed_field_bool = detail::packed_field_varint<std::string, bool>; + +/// Class for generating packed repeated enum fields. +using packed_field_enum = detail::packed_field_varint<std::string, int32_t>; + +/// Class for generating packed repeated int32 fields. +using packed_field_int32 = detail::packed_field_varint<std::string, int32_t>; + +/// Class for generating packed repeated sint32 fields. +using packed_field_sint32 = detail::packed_field_svarint<std::string, int32_t>; + +/// Class for generating packed repeated uint32 fields. +using packed_field_uint32 = detail::packed_field_varint<std::string, uint32_t>; + +/// Class for generating packed repeated int64 fields. +using packed_field_int64 = detail::packed_field_varint<std::string, int64_t>; + +/// Class for generating packed repeated sint64 fields. +using packed_field_sint64 = detail::packed_field_svarint<std::string, int64_t>; + +/// Class for generating packed repeated uint64 fields. +using packed_field_uint64 = detail::packed_field_varint<std::string, uint64_t>; + +/// Class for generating packed repeated fixed32 fields. +using packed_field_fixed32 = detail::packed_field_fixed<std::string, uint32_t>; + +/// Class for generating packed repeated sfixed32 fields. +using packed_field_sfixed32 = detail::packed_field_fixed<std::string, int32_t>; + +/// Class for generating packed repeated fixed64 fields. +using packed_field_fixed64 = detail::packed_field_fixed<std::string, uint64_t>; + +/// Class for generating packed repeated sfixed64 fields. +using packed_field_sfixed64 = detail::packed_field_fixed<std::string, int64_t>; + +/// Class for generating packed repeated float fields. +using packed_field_float = detail::packed_field_fixed<std::string, float>; + +/// Class for generating packed repeated double fields. +using packed_field_double = detail::packed_field_fixed<std::string, double>; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_WRITER_HPP diff --git a/thirdparty/protozero/include/protozero/types.hpp b/thirdparty/protozero/include/protozero/types.hpp new file mode 100644 index 000000000..ad9c362ff --- /dev/null +++ b/thirdparty/protozero/include/protozero/types.hpp @@ -0,0 +1,66 @@ +#ifndef PROTOZERO_TYPES_HPP +#define PROTOZERO_TYPES_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file types.hpp + * + * @brief Contains the declaration of low-level types used in the pbf format. + */ + +#include "config.hpp" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <string> +#include <utility> + +namespace protozero { + +/** + * The type used for field tags (field numbers). + */ +using pbf_tag_type = uint32_t; + +/** + * The type used to encode type information. + * See the table on + * https://developers.google.com/protocol-buffers/docs/encoding + */ +enum class pbf_wire_type : uint32_t { + varint = 0, // int32/64, uint32/64, sint32/64, bool, enum + fixed64 = 1, // fixed64, sfixed64, double + length_delimited = 2, // string, bytes, nested messages, packed repeated fields + fixed32 = 5, // fixed32, sfixed32, float + unknown = 99 // used for default setting in this library +}; + +/** + * Get the tag and wire type of the current field in one integer suitable + * for comparison with a switch statement. + * + * See pbf_reader.tag_and_type() for an example how to use this. + */ +template <typename T> +constexpr uint32_t tag_and_type(T tag, pbf_wire_type wire_type) noexcept { + return (static_cast<uint32_t>(static_cast<pbf_tag_type>(tag)) << 3U) | static_cast<uint32_t>(wire_type); +} + +/** + * The type used for length values, such as the length of a field. + */ +using pbf_length_type = uint32_t; + +} // end namespace protozero + +#endif // PROTOZERO_TYPES_HPP diff --git a/thirdparty/protozero/include/protozero/varint.hpp b/thirdparty/protozero/include/protozero/varint.hpp new file mode 100644 index 000000000..5c7fcec75 --- /dev/null +++ b/thirdparty/protozero/include/protozero/varint.hpp @@ -0,0 +1,244 @@ +#ifndef PROTOZERO_VARINT_HPP +#define PROTOZERO_VARINT_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file varint.hpp + * + * @brief Contains low-level varint and zigzag encoding and decoding functions. + */ + +#include "buffer_tmpl.hpp" +#include "exception.hpp" + +#include <cstdint> + +namespace protozero { + +/** + * The maximum length of a 64 bit varint. + */ +constexpr const int8_t max_varint_length = (sizeof(uint64_t) * 8 / 7) + 1; + +namespace detail { + + // from https://github.com/facebook/folly/blob/master/folly/Varint.h + inline uint64_t decode_varint_impl(const char** data, const char* end) { + const auto* begin = reinterpret_cast<const int8_t*>(*data); + const auto* iend = reinterpret_cast<const int8_t*>(end); + const int8_t* p = begin; + uint64_t val = 0; + + if (iend - begin >= max_varint_length) { // fast path + do { + int64_t b = *p++; + val = ((static_cast<uint64_t>(b) & 0x7fU) ); if (b >= 0) { break; } + b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 7U); if (b >= 0) { break; } + b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 14U); if (b >= 0) { break; } + b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 21U); if (b >= 0) { break; } + b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 28U); if (b >= 0) { break; } + b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 35U); if (b >= 0) { break; } + b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 42U); if (b >= 0) { break; } + b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 49U); if (b >= 0) { break; } + b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 56U); if (b >= 0) { break; } + b = *p++; val |= ((static_cast<uint64_t>(b) & 0x01U) << 63U); if (b >= 0) { break; } + throw varint_too_long_exception{}; + } while (false); + } else { + unsigned int shift = 0; + while (p != iend && *p < 0) { + val |= (static_cast<uint64_t>(*p++) & 0x7fU) << shift; + shift += 7; + } + if (p == iend) { + throw end_of_buffer_exception{}; + } + val |= static_cast<uint64_t>(*p++) << shift; + } + + *data = reinterpret_cast<const char*>(p); + return val; + } + +} // end namespace detail + +/** + * Decode a 64 bit varint. + * + * Strong exception guarantee: if there is an exception the data pointer will + * not be changed. + * + * @param[in,out] data Pointer to pointer to the input data. After the function + * returns this will point to the next data to be read. + * @param[in] end Pointer one past the end of the input data. + * @returns The decoded integer + * @throws varint_too_long_exception if the varint is longer then the maximum + * length that would fit in a 64 bit int. Usually this means your data + * is corrupted or you are trying to read something as a varint that + * isn't. + * @throws end_of_buffer_exception if the *end* of the buffer was reached + * before the end of the varint. + */ +inline uint64_t decode_varint(const char** data, const char* end) { + // If this is a one-byte varint, decode it here. + if (end != *data && ((static_cast<uint64_t>(**data) & 0x80U) == 0)) { + const auto val = static_cast<uint64_t>(**data); + ++(*data); + return val; + } + // If this varint is more than one byte, defer to complete implementation. + return detail::decode_varint_impl(data, end); +} + +/** + * Skip over a varint. + * + * Strong exception guarantee: if there is an exception the data pointer will + * not be changed. + * + * @param[in,out] data Pointer to pointer to the input data. After the function + * returns this will point to the next data to be read. + * @param[in] end Pointer one past the end of the input data. + * @throws end_of_buffer_exception if the *end* of the buffer was reached + * before the end of the varint. + */ +inline void skip_varint(const char** data, const char* end) { + const auto* begin = reinterpret_cast<const int8_t*>(*data); + const auto* iend = reinterpret_cast<const int8_t*>(end); + const int8_t* p = begin; + + while (p != iend && *p < 0) { + ++p; + } + + if (p - begin >= max_varint_length) { + throw varint_too_long_exception{}; + } + + if (p == iend) { + throw end_of_buffer_exception{}; + } + + ++p; + + *data = reinterpret_cast<const char*>(p); +} + +/** + * Varint encode a 64 bit integer. + * + * @tparam T An output iterator type. + * @param data Output iterator the varint encoded value will be written to + * byte by byte. + * @param value The integer that will be encoded. + * @returns the number of bytes written + * @throws Any exception thrown by increment or dereference operator on data. + * @deprecated Use add_varint_to_buffer() instead. + */ +template <typename T> +inline int write_varint(T data, uint64_t value) { + int n = 1; + + while (value >= 0x80U) { + *data++ = static_cast<char>((value & 0x7fU) | 0x80U); + value >>= 7U; + ++n; + } + *data = static_cast<char>(value); + + return n; +} + +/** + * Varint encode a 64 bit integer. + * + * @tparam TBuffer A buffer type. + * @param buffer Output buffer the varint will be written to. + * @param value The integer that will be encoded. + * @throws Any exception thrown by calling the buffer_push_back() function. + */ +template <typename TBuffer> +inline void add_varint_to_buffer(TBuffer* buffer, uint64_t value) { + while (value >= 0x80U) { + buffer_customization<TBuffer>::push_back(buffer, static_cast<char>((value & 0x7fU) | 0x80U)); + value >>= 7U; + } + buffer_customization<TBuffer>::push_back(buffer, static_cast<char>(value)); +} + +/** + * Varint encode a 64 bit integer. + * + * @param data Where to add the varint. There must be enough space available! + * @param value The integer that will be encoded. + * @returns the number of bytes written + */ +inline int add_varint_to_buffer(char* data, uint64_t value) noexcept { + int n = 1; + + while (value >= 0x80U) { + *data++ = static_cast<char>((value & 0x7fU) | 0x80U); + value >>= 7U; + ++n; + } + *data = static_cast<char>(value); + + return n; +} + +/** + * Get the length of the varint the specified value would produce. + * + * @param value The integer to be encoded. + * @returns the number of bytes the varint would have if we created it. + */ +inline int length_of_varint(uint64_t value) noexcept { + int n = 1; + + while (value >= 0x80U) { + value >>= 7U; + ++n; + } + + return n; +} + +/** + * ZigZag encodes a 32 bit integer. + */ +constexpr uint32_t encode_zigzag32(int32_t value) noexcept { + return (static_cast<uint32_t>(value) << 1U) ^ static_cast<uint32_t>(-static_cast<int32_t>(static_cast<uint32_t>(value) >> 31U)); +} + +/** + * ZigZag encodes a 64 bit integer. + */ +constexpr uint64_t encode_zigzag64(int64_t value) noexcept { + return (static_cast<uint64_t>(value) << 1U) ^ static_cast<uint64_t>(-static_cast<int64_t>(static_cast<uint64_t>(value) >> 63U)); +} + +/** + * Decodes a 32 bit ZigZag-encoded integer. + */ +constexpr int32_t decode_zigzag32(uint32_t value) noexcept { + return static_cast<int32_t>((value >> 1U) ^ static_cast<uint32_t>(-static_cast<int32_t>(value & 1U))); +} + +/** + * Decodes a 64 bit ZigZag-encoded integer. + */ +constexpr int64_t decode_zigzag64(uint64_t value) noexcept { + return static_cast<int64_t>((value >> 1U) ^ static_cast<uint64_t>(-static_cast<int64_t>(value & 1U))); +} + +} // end namespace protozero + +#endif // PROTOZERO_VARINT_HPP diff --git a/thirdparty/protozero/include/protozero/version.hpp b/thirdparty/protozero/include/protozero/version.hpp new file mode 100644 index 000000000..760870f97 --- /dev/null +++ b/thirdparty/protozero/include/protozero/version.hpp @@ -0,0 +1,34 @@ +#ifndef PROTOZERO_VERSION_HPP +#define PROTOZERO_VERSION_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file version.hpp + * + * @brief Contains macros defining the protozero version. + */ + +/// The major version number +#define PROTOZERO_VERSION_MAJOR 1 + +/// The minor version number +#define PROTOZERO_VERSION_MINOR 8 + +/// The patch number +#define PROTOZERO_VERSION_PATCH 1 + +/// The complete version number +#define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH) + +/// Version number as string +#define PROTOZERO_VERSION_STRING "1.8.1" + +#endif // PROTOZERO_VERSION_HPP diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua index d745a5109..c9acfa018 100644 --- a/thirdparty/xmake.lua +++ b/thirdparty/xmake.lua @@ -43,3 +43,9 @@ target('rpmalloc') add_files("rpmalloc/rpmalloc.c") add_headerfiles("rpmalloc/**.h") add_includedirs("rpmalloc", {public=true}) + +target('protozero') + set_kind('headeronly') + set_group('thirdparty') + add_headerfiles("protozero/**.hpp") + add_includedirs("protozero/include", {public=true}) @@ -97,7 +97,8 @@ if is_os("windows") then "NOMINMAX", -- stop Windows SDK defining 'min' and 'max' "NOGDI", -- otherwise Windows.h defines 'GetObject' "WIN32_LEAN_AND_MEAN", -- cut down Windows.h - "_WIN32_WINNT=0x0A00" + "_WIN32_WINNT=0x0A00", + "_WINSOCK_DEPRECATED_NO_WARNINGS" -- let us use the ANSI functions ) -- Make builds more deterministic and portable add_cxxflags("/d1trimfile:$(curdir)\\") -- eliminates the base path from __FILE__ paths @@ -202,14 +203,15 @@ set_symbols("debug") includes("thirdparty") includes("src/transports") includes("src/zenbase") -includes("src/zencore", "src/zencore-test") -includes("src/zenhttp", "src/zenhttp-test") -includes("src/zennet", "src/zennet-test") -includes("src/zenremotestore", "src/zenremotestore-test") -includes("src/zenstore", "src/zenstore-test") -includes("src/zenutil", "src/zenutil-test") +includes("src/zencore", "src/zencore-test") +includes("src/zenhttp", "src/zenhttp-test") +includes("src/zennet", "src/zennet-test") +includes("src/zenremotestore", "src/zenremotestore-test") +includes("src/zenstore", "src/zenstore-test") +includes("src/zentelemetry", "src/zentelemetry-test") +includes("src/zenutil", "src/zenutil-test") includes("src/zenvfs") -includes("src/zenserver", "src/zenserver-test") +includes("src/zenserver", "src/zenserver-test") includes("src/zen") includes("src/zentest-appstub") |