aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2025-10-22 17:57:29 +0200
committerGitHub Enterprise <[email protected]>2025-10-22 17:57:29 +0200
commit5c139e2d8a260544bc5e730de0440edbab4b0f03 (patch)
treeb477208925fe3b373d4833460b90d61a8051cf05
parent5.7.7-pre3 (diff)
downloadzen-5c139e2d8a260544bc5e730de0440edbab4b0f03.tar.xz
zen-5c139e2d8a260544bc5e730de0440edbab4b0f03.zip
add support for OTLP logging/tracing (#599)
- adds `zentelemetry` project which houses new functionality for serializing logs and traces in OpenTelemetry Protocol format (OTLP) - moved existing stats functionality from `zencore` to `zentelemetry` - adds `TRefCounted<T>` for vtable-less refcounting - adds `MemoryArena` class which allows for linear allocation of memory from chunks - adds `protozero` which is used to encode OTLP protobuf messages
-rw-r--r--src/zenbase/include/zenbase/refcount.h51
-rw-r--r--src/zencore/include/zencore/iobuffer.h3
-rw-r--r--src/zencore/include/zencore/memory/memoryarena.h104
-rw-r--r--src/zencore/memory/memoryarena.cpp126
-rw-r--r--src/zencore/trace.cpp1
-rw-r--r--src/zencore/zencore.cpp2
-rw-r--r--src/zenhttp/httpserver.cpp5
-rw-r--r--src/zenhttp/include/zenhttp/httptest.h2
-rw-r--r--src/zenhttp/xmake.lua2
-rw-r--r--src/zenserver/diag/logging.cpp12
-rw-r--r--src/zenserver/diag/otlphttp.cpp83
-rw-r--r--src/zenserver/diag/otlphttp.h64
-rw-r--r--src/zenserver/storage/buildstore/httpbuildstore.h2
-rw-r--r--src/zenserver/storage/cache/httpstructuredcache.h2
-rw-r--r--src/zenserver/storage/projectstore/httpprojectstore.h2
-rw-r--r--src/zenserver/storage/upstream/upstreamcache.cpp2
-rw-r--r--src/zenserver/storage/upstream/upstreamcache.h2
-rw-r--r--src/zenserver/storage/workspaces/httpworkspaces.h2
-rw-r--r--src/zenserver/xmake.lua3
-rw-r--r--src/zenstore/cidstore.cpp2
-rw-r--r--src/zenstore/include/zenstore/cache/cachedisklayer.h2
-rw-r--r--src/zenstore/include/zenstore/cache/structuredcachestore.h2
-rw-r--r--src/zenstore/include/zenstore/cache/upstreamcacheclient.h2
-rw-r--r--src/zenstore/include/zenstore/cidstore.h2
-rw-r--r--src/zentelemetry-test/xmake.lua9
-rw-r--r--src/zentelemetry-test/zentelemetry-test.cpp42
-rw-r--r--src/zentelemetry/include/zentelemetry/otlpencoder.h66
-rw-r--r--src/zentelemetry/include/zentelemetry/otlptrace.h268
-rw-r--r--src/zentelemetry/include/zentelemetry/stats.h (renamed from src/zencore/include/zencore/stats.h)2
-rw-r--r--src/zentelemetry/include/zentelemetry/zentelemetry.h9
-rw-r--r--src/zentelemetry/otellogprotozero.h287
-rw-r--r--src/zentelemetry/otelmetricsprotozero.h953
-rw-r--r--src/zentelemetry/otelprotozero.h166
-rw-r--r--src/zentelemetry/oteltraceprotozero.h474
-rw-r--r--src/zentelemetry/otlpencoder.cpp478
-rw-r--r--src/zentelemetry/otlptrace.cpp398
-rw-r--r--src/zentelemetry/stats.cpp (renamed from src/zencore/stats.cpp)2
-rw-r--r--src/zentelemetry/xmake.lua10
-rw-r--r--src/zentelemetry/zentelemetry.cpp19
-rw-r--r--thirdparty/protozero/CHANGELOG.md457
-rw-r--r--thirdparty/protozero/LICENSE.from_folly177
-rw-r--r--thirdparty/protozero/LICENSE.md26
-rw-r--r--thirdparty/protozero/README.md155
-rw-r--r--thirdparty/protozero/include/protozero/basic_pbf_builder.hpp266
-rw-r--r--thirdparty/protozero/include/protozero/basic_pbf_writer.hpp1054
-rw-r--r--thirdparty/protozero/include/protozero/buffer_fixed.hpp222
-rw-r--r--thirdparty/protozero/include/protozero/buffer_string.hpp78
-rw-r--r--thirdparty/protozero/include/protozero/buffer_tmpl.hpp113
-rw-r--r--thirdparty/protozero/include/protozero/buffer_vector.hpp78
-rw-r--r--thirdparty/protozero/include/protozero/byteswap.hpp108
-rw-r--r--thirdparty/protozero/include/protozero/config.hpp48
-rw-r--r--thirdparty/protozero/include/protozero/data_view.hpp236
-rw-r--r--thirdparty/protozero/include/protozero/exception.hpp101
-rw-r--r--thirdparty/protozero/include/protozero/iterators.hpp481
-rw-r--r--thirdparty/protozero/include/protozero/pbf_builder.hpp32
-rw-r--r--thirdparty/protozero/include/protozero/pbf_message.hpp184
-rw-r--r--thirdparty/protozero/include/protozero/pbf_reader.hpp977
-rw-r--r--thirdparty/protozero/include/protozero/pbf_writer.hpp76
-rw-r--r--thirdparty/protozero/include/protozero/types.hpp66
-rw-r--r--thirdparty/protozero/include/protozero/varint.hpp244
-rw-r--r--thirdparty/protozero/include/protozero/version.hpp34
-rw-r--r--thirdparty/xmake.lua6
-rw-r--r--xmake.lua18
63 files changed, 8872 insertions, 28 deletions
diff --git a/src/zenbase/include/zenbase/refcount.h b/src/zenbase/include/zenbase/refcount.h
index 6ad49cba2..40ad7bca5 100644
--- a/src/zenbase/include/zenbase/refcount.h
+++ b/src/zenbase/include/zenbase/refcount.h
@@ -10,6 +10,9 @@ namespace zen {
/**
* Helper base class for reference counted objects using intrusive reference counting
+ *
+ * When the reference count reaches zero, the object deletes itself. This class relies
+ * on having a virtual destructor to ensure proper cleanup of derived classes.
*/
class RefCounted
{
@@ -17,7 +20,7 @@ public:
RefCounted() = default;
virtual ~RefCounted() = default;
- inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); }
+ inline uint32_t AddRef() const noexcept { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); }
inline uint32_t Release() const
{
const uint32_t RefCount = AtomicDecrement(const_cast<RefCounted*>(this)->m_RefCount);
@@ -43,6 +46,48 @@ private:
};
/**
+ * Template helper base class for reference counted objects using intrusive reference counting.
+ *
+ * NOTE: Unlike RefCounted, this class deletes the derived type when the reference count reaches zero.
+ * It has no virtual destructor, so it's important that you either don't derive from it further,
+ * or ensure that the derived class has a virtual destructor.
+ */
+
+template<typename T>
+class TRefCounted
+{
+public:
+ TRefCounted() = default;
+ ~TRefCounted() = default;
+
+ inline uint32_t AddRef() const noexcept { return AtomicIncrement(const_cast<TRefCounted<T>*>(this)->m_RefCount); }
+ inline uint32_t Release() const
+ {
+ const uint32_t RefCount = AtomicDecrement(const_cast<TRefCounted<T>*>(this)->m_RefCount);
+ if (RefCount == 0)
+ {
+ const_cast<T*>(static_cast<const T*>(this))->DeleteThis();
+ }
+ return RefCount;
+ }
+
+ // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it
+
+ TRefCounted(const TRefCounted&) = delete;
+ TRefCounted(TRefCounted&&) = delete;
+ TRefCounted& operator=(const TRefCounted&) = delete;
+ TRefCounted& operator=(TRefCounted&&) = delete;
+
+protected:
+ inline uint32_t RefCount() const { return m_RefCount; }
+
+ void DeleteThis() const noexcept { delete static_cast<const T*>(this); }
+
+private:
+ uint32_t m_RefCount = 0;
+};
+
+/**
* Smart pointer for classes derived from RefCounted
*/
@@ -127,7 +172,8 @@ class Ref
{
public:
inline Ref() = default;
- inline Ref(const Ref& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); }
+ inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; }
+ inline Ref(const Ref& Rhs) noexcept : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); }
inline explicit Ref(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); }
inline ~Ref() { m_Ref && m_Ref->Release(); }
@@ -170,7 +216,6 @@ public:
}
return *this;
}
- inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; }
private:
T* m_Ref = nullptr;
diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h
index 63779407e..1b2d382ee 100644
--- a/src/zencore/include/zencore/iobuffer.h
+++ b/src/zencore/include/zencore/iobuffer.h
@@ -32,6 +32,7 @@ enum class ZenContentType : uint8_t
kPNG = 12,
kIcon = 13,
kXML = 14,
+ kProtobuf = 15,
kCOUNT
};
@@ -73,6 +74,8 @@ ToString(ZenContentType ContentType)
return "icon"sv;
case ZenContentType::kXML:
return "xml"sv;
+ case ZenContentType::kProtobuf:
+ return "protobuf"sv;
}
}
diff --git a/src/zencore/include/zencore/memory/memoryarena.h b/src/zencore/include/zencore/memory/memoryarena.h
new file mode 100644
index 000000000..551415aac
--- /dev/null
+++ b/src/zencore/include/zencore/memory/memoryarena.h
@@ -0,0 +1,104 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/thread.h>
+#include <vector>
+
+namespace zen {
+
+/**
+ * Chunked linear memory allocator
+ *
+ * Supports fast allocation of many small objects with minimal overhead.
+ * All allocations are freed when the arena is destroyed, therefore there
+ * is no support for individual deallocation.
+ *
+ * For convenience, we include operator new/delete overloads below which
+ * take a MemoryArena reference as a placement argument.
+ *
+ * This allocator is thread-safe.
+ */
+class MemoryArena
+{
+public:
+ MemoryArena() = default;
+ ~MemoryArena();
+
+ void* AllocateAligned(size_t ByteCount, size_t align);
+ void* AllocateAlignedWithOffset(size_t ByteCount, size_t align, size_t offset);
+ void* Allocate(size_t Size);
+ const char* DuplicateString(std::string_view Str);
+
+ MemoryArena(const MemoryArena&) = delete;
+ MemoryArena& operator=(const MemoryArena&) = delete;
+
+private:
+ static constexpr size_t ChunkSize = 16 * 1024;
+ // TODO: should just chain the memory blocks together and avoid this
+ // vector altogether, saving us a memory allocation
+ std::vector<uint8_t*> m_Chunks;
+ uint8_t* m_CurrentChunk = nullptr;
+ size_t m_CurrentOffset = 0;
+ RwLock m_Lock;
+};
+
+// Allocator suitable for use with EASTL
+
+struct ArenaAlloc
+{
+ ArenaAlloc(const char* name_opt = nullptr) = delete;
+ ArenaAlloc(MemoryArena& Arena) : m_Arena(&Arena) {}
+
+ inline void* allocate(size_t bytes, int flags = 0)
+ {
+ ZEN_UNUSED(flags);
+ return m_Arena->Allocate(bytes);
+ }
+
+ inline void* allocate(size_t bytes, size_t align, size_t offset, int flags = 0)
+ {
+ ZEN_UNUSED(flags);
+ if (offset == 0)
+ {
+ return m_Arena->AllocateAligned(bytes, align);
+ }
+ else
+ {
+ return m_Arena->AllocateAlignedWithOffset(bytes, align, offset);
+ }
+ }
+
+ void deallocate(void* p, size_t n) { ZEN_UNUSED(p, n); }
+
+private:
+ MemoryArena* m_Arena = nullptr;
+};
+
+} // namespace zen
+
+inline void*
+operator new(size_t Size, zen::MemoryArena& Arena)
+{
+ return Arena.Allocate(Size);
+}
+
+inline void
+operator delete(void* Ptr, zen::MemoryArena& Arena)
+{
+ // Arena will clean up all allocations when it's destroyed
+ ZEN_UNUSED(Ptr, Arena);
+}
+
+inline void*
+operator new[](size_t Size, zen::MemoryArena& Arena)
+{
+ return Arena.Allocate(Size);
+}
+
+inline void
+operator delete[](void* Ptr, zen::MemoryArena& Arena)
+{
+ // Arena will clean up all allocations when it's destroyed
+ ZEN_UNUSED(Ptr, Arena);
+}
diff --git a/src/zencore/memory/memoryarena.cpp b/src/zencore/memory/memoryarena.cpp
new file mode 100644
index 000000000..9c907a66d
--- /dev/null
+++ b/src/zencore/memory/memoryarena.cpp
@@ -0,0 +1,126 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/memory/memoryarena.h>
+
+namespace zen {
+
+MemoryArena::~MemoryArena()
+{
+ for (auto Chunk : m_Chunks)
+ delete[] Chunk;
+}
+
+void*
+MemoryArena::AllocateAligned(size_t ByteCount, size_t align)
+{
+ if (ByteCount == 0)
+ {
+ return nullptr;
+ }
+
+ void* Ptr = nullptr;
+
+ m_Lock.WithExclusiveLock([&] {
+ size_t AlignedOffset = (m_CurrentOffset + (align - 1)) & ~(align - 1);
+
+ if (m_CurrentChunk == nullptr || AlignedOffset + ByteCount > ChunkSize)
+ {
+ uint8_t* NewChunk = new uint8_t[ChunkSize];
+ if (!NewChunk)
+ {
+ return;
+ }
+
+ m_Chunks.push_back(NewChunk);
+ m_CurrentChunk = NewChunk;
+ AlignedOffset = 0;
+ }
+
+ Ptr = m_CurrentChunk + AlignedOffset;
+ m_CurrentOffset = AlignedOffset + ByteCount;
+ });
+
+ return Ptr;
+}
+
+void*
+MemoryArena::AllocateAlignedWithOffset(size_t ByteCount, size_t align, size_t offset)
+{
+ if (ByteCount == 0)
+ {
+ return nullptr;
+ }
+
+ void* Ptr = nullptr;
+
+ m_Lock.WithExclusiveLock([&] {
+ size_t AlignedOffset = (m_CurrentOffset + (align - 1) + offset) & ~(align - 1);
+
+ if (m_CurrentChunk == nullptr || AlignedOffset + ByteCount > ChunkSize)
+ {
+ uint8_t* NewChunk = new uint8_t[ChunkSize];
+ if (!NewChunk)
+ {
+ return;
+ }
+
+ m_Chunks.push_back(NewChunk);
+ m_CurrentChunk = NewChunk;
+ AlignedOffset = offset;
+ }
+
+ Ptr = m_CurrentChunk + AlignedOffset;
+ m_CurrentOffset = AlignedOffset + ByteCount;
+ });
+
+ return Ptr;
+}
+
+void*
+MemoryArena::Allocate(size_t Size)
+{
+ if (Size == 0)
+ {
+ return nullptr;
+ }
+
+ void* Ptr = nullptr;
+ constexpr size_t Alignment = alignof(std::max_align_t);
+
+ m_Lock.WithExclusiveLock([&] {
+ size_t AlignedOffset = (m_CurrentOffset + Alignment - 1) & ~(Alignment - 1);
+
+ if (m_CurrentChunk == nullptr || AlignedOffset + Size > ChunkSize)
+ {
+ uint8_t* NewChunk = new uint8_t[ChunkSize];
+ if (!NewChunk)
+ {
+ return;
+ }
+
+ m_Chunks.push_back(NewChunk);
+ m_CurrentChunk = NewChunk;
+ AlignedOffset = 0;
+ }
+
+ Ptr = m_CurrentChunk + AlignedOffset;
+ m_CurrentOffset = AlignedOffset + Size;
+ });
+
+ return Ptr;
+}
+
+const char*
+MemoryArena::DuplicateString(std::string_view Str)
+{
+ const size_t Len = Str.size();
+ char* NewStr = static_cast<char*>(Allocate(Len + 1));
+ if (NewStr)
+ {
+ memcpy(NewStr, Str.data(), Len);
+ NewStr[Len] = '\0';
+ }
+ return NewStr;
+}
+
+} // namespace zen
diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp
index fe8fb9a5d..87035554f 100644
--- a/src/zencore/trace.cpp
+++ b/src/zencore/trace.cpp
@@ -9,6 +9,7 @@
# include <zencore/logging.h>
# define TRACE_IMPLEMENT 1
+# undef _WINSOCK_DEPRECATED_NO_WARNINGS
# include <zencore/trace.h>
# include <zencore/memory/fmalloc.h>
diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp
index b78991918..4ff79edc7 100644
--- a/src/zencore/zencore.cpp
+++ b/src/zencore/zencore.cpp
@@ -26,7 +26,6 @@
#include <zencore/parallelwork.h>
#include <zencore/process.h>
#include <zencore/sha1.h>
-#include <zencore/stats.h>
#include <zencore/stream.h>
#include <zencore/string.h>
#include <zencore/thread.h>
@@ -267,7 +266,6 @@ zencore_forcelinktests()
zen::process_forcelink();
zen::refcount_forcelink();
zen::sha1_forcelink();
- zen::stats_forcelink();
zen::stream_forcelink();
zen::string_forcelink();
zen::thread_forcelink();
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index 2c063d646..f48c22367 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -86,6 +86,9 @@ MapContentTypeToString(HttpContentType ContentType)
case HttpContentType::kXML:
return "application/xml"sv;
+
+ case HttpContentType::kProtobuf:
+ return "application/x-protobuf"sv;
}
}
@@ -119,6 +122,7 @@ static constinit uint32_t HashImagePng = HashStringDjb2("image/png"sv);
static constinit uint32_t HashIcon = HashStringDjb2("ico"sv);
static constinit uint32_t HashImageIcon = HashStringDjb2("image/x-icon"sv);
static constinit uint32_t HashXml = HashStringDjb2("application/xml"sv);
+static constinit uint32_t HashProtobuf = HashStringDjb2("application/x-protobuf"sv);
std::once_flag InitContentTypeLookup;
@@ -153,6 +157,7 @@ struct HashedTypeEntry
{HashIcon, HttpContentType::kIcon},
{HashImageIcon, HttpContentType::kIcon},
{HashXml, HttpContentType::kXML},
+ {HashProtobuf, HttpContentType::kProtobuf},
// clang-format on
};
diff --git a/src/zenhttp/include/zenhttp/httptest.h b/src/zenhttp/include/zenhttp/httptest.h
index afe71fbce..608462809 100644
--- a/src/zenhttp/include/zenhttp/httptest.h
+++ b/src/zenhttp/include/zenhttp/httptest.h
@@ -3,8 +3,8 @@
#pragma once
#include <zencore/logging.h>
-#include <zencore/stats.h>
#include <zenhttp/httpserver.h>
+#include <zentelemetry/stats.h>
#include <atomic>
#include <unordered_map>
diff --git a/src/zenhttp/xmake.lua b/src/zenhttp/xmake.lua
index b6ffbe467..af4064012 100644
--- a/src/zenhttp/xmake.lua
+++ b/src/zenhttp/xmake.lua
@@ -7,7 +7,7 @@ target('zenhttp')
add_files("**.cpp")
add_files("servers/httpsys.cpp", {unity_ignored=true})
add_includedirs("include", {public=true})
- add_deps("zencore", "transport-sdk")
+ add_deps("zencore", "zentelemetry", "transport-sdk")
add_packages(
"vcpkg::asio",
"vcpkg::cpr",
diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp
index 50cf62274..90af79651 100644
--- a/src/zenserver/diag/logging.cpp
+++ b/src/zenserver/diag/logging.cpp
@@ -12,6 +12,8 @@
#include <zenutil/logging.h>
#include <zenutil/logging/rotatingfilesink.h>
+#include "otlphttp.h"
+
ZEN_THIRD_PARTY_INCLUDES_START
#include <spdlog/spdlog.h>
ZEN_THIRD_PARTY_INCLUDES_END
@@ -73,6 +75,16 @@ InitializeServerLogging(const ZenServerConfig& InOptions)
spdlog::apply_logger_env_levels(ZenClientLogger);
spdlog::register_logger(ZenClientLogger);
+ //
+
+#if ZEN_WITH_OTEL
+ if (false)
+ {
+ auto OtelSink = std::make_shared<zen::logging::OtelHttpProtobufSink>("http://signoz.localdomain:4318");
+ zen::logging::Default().SpdLogger->sinks().push_back(std::move(OtelSink));
+ }
+#endif
+
FinishInitializeLogging(LogOptions);
const zen::Oid ServerSessionId = zen::GetSessionId();
diff --git a/src/zenserver/diag/otlphttp.cpp b/src/zenserver/diag/otlphttp.cpp
new file mode 100644
index 000000000..d62ccccb6
--- /dev/null
+++ b/src/zenserver/diag/otlphttp.cpp
@@ -0,0 +1,83 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "otlphttp.h"
+
+#include <zencore/config.h>
+#include <zencore/process.h>
+#include <zencore/session.h>
+#include <zencore/system.h>
+#include <zentelemetry/otlpencoder.h>
+#include <protozero/buffer_string.hpp>
+#include <protozero/pbf_builder.hpp>
+
+#if ZEN_WITH_OTEL
+
+namespace zen::logging {
+
+//////////////////////////////////////////////////////////////////////////
+
+OtelHttpProtobufSink::OtelHttpProtobufSink(const std::string_view& Uri) : m_OtelHttp(Uri)
+{
+ m_Encoder.AddResourceAttribute("service.name", "zenserver");
+ m_Encoder.AddResourceAttribute("service.instance.id", GetSessionIdString());
+ m_Encoder.AddResourceAttribute("service.namespace", "zen");
+ m_Encoder.AddResourceAttribute("service.version", ZEN_CFG_VERSION);
+ m_Encoder.AddResourceAttribute("host.name", GetMachineName());
+ m_Encoder.AddResourceAttribute("session.id", GetSessionIdString());
+ m_Encoder.AddResourceAttribute("process.id", zen::GetCurrentProcessId());
+
+ m_TraceRecorder = new TraceRecorder(this);
+ otel::SetTraceRecorder(m_TraceRecorder);
+}
+
+OtelHttpProtobufSink::~OtelHttpProtobufSink()
+{
+ otel::SetTraceRecorder({});
+}
+
+void
+OtelHttpProtobufSink::RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans)
+{
+ std::string Data = m_Encoder.FormatOtelTrace(Trace, Spans);
+
+ IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
+ Payload.SetContentType(ZenContentType::kProtobuf);
+
+ auto Result = m_OtelHttp.Post("/v1/traces", Payload);
+}
+
+void
+OtelHttpProtobufSink::TraceRecorder::RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans)
+{
+ m_Sink->RecordSpans(Trace, Spans);
+}
+
+void
+OtelHttpProtobufSink::log(const spdlog::details::log_msg& Msg)
+{
+ {
+ std::string Data = m_Encoder.FormatOtelProtobuf(Msg);
+
+ IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
+ Payload.SetContentType(ZenContentType::kProtobuf);
+
+ auto Result = m_OtelHttp.Post("/v1/logs", Payload);
+ }
+
+ {
+ std::string Data = m_Encoder.FormatOtelMetrics();
+
+ IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
+ Payload.SetContentType(ZenContentType::kProtobuf);
+
+ auto Result = m_OtelHttp.Post("/v1/metrics", Payload);
+ }
+}
+void
+OtelHttpProtobufSink::flush()
+{
+}
+
+} // namespace zen::logging
+
+#endif
diff --git a/src/zenserver/diag/otlphttp.h b/src/zenserver/diag/otlphttp.h
new file mode 100644
index 000000000..2281bdcc0
--- /dev/null
+++ b/src/zenserver/diag/otlphttp.h
@@ -0,0 +1,64 @@
+
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <spdlog/sinks/sink.h>
+#include <zencore/zencore.h>
+#include <zenhttp/httpclient.h>
+#include <zentelemetry/otlpencoder.h>
+#include <zentelemetry/otlptrace.h>
+
+#if ZEN_WITH_OTEL
+
+namespace zen::logging {
+
+/**
+ * OTLP/HTTP sink for spdlog
+ *
+ * Sends log messages and traces to an OpenTelemetry collector via OTLP over HTTP
+ */
+
+class OtelHttpProtobufSink : public spdlog::sinks::sink
+{
+public:
+ // Note that this URI should be the base URI of the OTLP HTTP endpoint, e.g.
+ // "http://otel-collector:4318"
+ OtelHttpProtobufSink(const std::string_view& Uri);
+ ~OtelHttpProtobufSink();
+
+ OtelHttpProtobufSink(const OtelHttpProtobufSink&) = delete;
+ OtelHttpProtobufSink& operator=(const OtelHttpProtobufSink&) = delete;
+
+private:
+ virtual void log(const spdlog::details::log_msg& Msg) override;
+ virtual void flush() override;
+ virtual void set_pattern(const std::string& pattern) override { ZEN_UNUSED(pattern); }
+ virtual void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) override { ZEN_UNUSED(sink_formatter); }
+
+ void RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans);
+
+ // This is just a thin wrapper to call back into the sink while participating in
+ // reference counting from the OTEL trace back-end
+ class TraceRecorder : public zen::otel::TraceRecorder
+ {
+ public:
+ TraceRecorder(OtelHttpProtobufSink* InSink) : m_Sink(InSink) {}
+
+ private:
+ TraceRecorder(const TraceRecorder&) = delete;
+ TraceRecorder& operator=(const TraceRecorder&) = delete;
+
+ virtual void RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans) override;
+
+ OtelHttpProtobufSink* m_Sink;
+ };
+
+ HttpClient m_OtelHttp;
+ OtlpEncoder m_Encoder;
+ Ref<TraceRecorder> m_TraceRecorder;
+};
+
+} // namespace zen::logging
+
+#endif \ No newline at end of file
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h
index 50cb5db12..e10986411 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.h
+++ b/src/zenserver/storage/buildstore/httpbuildstore.h
@@ -2,10 +2,10 @@
#pragma once
-#include <zencore/stats.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstats.h>
#include <zenhttp/httpstatus.h>
+#include <zentelemetry/stats.h>
#include <filesystem>
diff --git a/src/zenserver/storage/cache/httpstructuredcache.h b/src/zenserver/storage/cache/httpstructuredcache.h
index a157148c9..5a795c215 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.h
+++ b/src/zenserver/storage/cache/httpstructuredcache.h
@@ -2,12 +2,12 @@
#pragma once
-#include <zencore/stats.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstats.h>
#include <zenhttp/httpstatus.h>
#include <zenstore/cache/cache.h>
#include <zenstore/cache/cacherpc.h>
+#include <zentelemetry/stats.h>
#include <zenutil/openprocesscache.h>
#include <memory>
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h
index f0a0bcfa1..f6fe63614 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.h
+++ b/src/zenserver/storage/projectstore/httpprojectstore.h
@@ -2,11 +2,11 @@
#pragma once
-#include <zencore/stats.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstats.h>
#include <zenhttp/httpstatus.h>
#include <zenstore/cidstore.h>
+#include <zentelemetry/stats.h>
namespace zen {
diff --git a/src/zenserver/storage/upstream/upstreamcache.cpp b/src/zenserver/storage/upstream/upstreamcache.cpp
index f7ae5f973..6c489c5d3 100644
--- a/src/zenserver/storage/upstream/upstreamcache.cpp
+++ b/src/zenserver/storage/upstream/upstreamcache.cpp
@@ -9,10 +9,10 @@
#include <zencore/compactbinarypackage.h>
#include <zencore/compactbinaryvalidation.h>
#include <zencore/fmtutils.h>
-#include <zencore/stats.h>
#include <zencore/stream.h>
#include <zencore/timer.h>
#include <zencore/trace.h>
+#include <zentelemetry/stats.h>
#include <zenhttp/httpclientauth.h>
#include <zenhttp/packageformat.h>
diff --git a/src/zenserver/storage/upstream/upstreamcache.h b/src/zenserver/storage/upstream/upstreamcache.h
index d5d61c8d9..c0c8a7ff9 100644
--- a/src/zenserver/storage/upstream/upstreamcache.h
+++ b/src/zenserver/storage/upstream/upstreamcache.h
@@ -6,10 +6,10 @@
#include <zencore/compress.h>
#include <zencore/iobuffer.h>
#include <zencore/iohash.h>
-#include <zencore/stats.h>
#include <zencore/zencore.h>
#include <zenstore/cache/cache.h>
#include <zenstore/cache/upstreamcacheclient.h>
+#include <zentelemetry/stats.h>
#include <atomic>
#include <chrono>
diff --git a/src/zenserver/storage/workspaces/httpworkspaces.h b/src/zenserver/storage/workspaces/httpworkspaces.h
index 89a8e8bdc..888a34b4d 100644
--- a/src/zenserver/storage/workspaces/httpworkspaces.h
+++ b/src/zenserver/storage/workspaces/httpworkspaces.h
@@ -2,10 +2,10 @@
#pragma once
-#include <zencore/stats.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstats.h>
#include <zenhttp/httpstatus.h>
+#include <zentelemetry/stats.h>
namespace zen {
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index 57105045d..483bfd5aa 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -7,6 +7,7 @@ target("zenserver")
"zennet",
"zenremotestore",
"zenstore",
+ "zentelemetry",
"zenutil",
"zenvfs")
add_headerfiles("**.h")
@@ -17,6 +18,8 @@ target("zenserver")
add_includedirs(".")
set_symbols("debug")
+ add_deps("protozero")
+
if is_mode("release") then
set_optimize("fastest")
end
diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp
index ae1b59dc0..52d5df061 100644
--- a/src/zenstore/cidstore.cpp
+++ b/src/zenstore/cidstore.cpp
@@ -6,9 +6,9 @@
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
-#include <zencore/stats.h>
#include <zencore/string.h>
#include <zenstore/scrubcontext.h>
+#include <zentelemetry/stats.h>
#include "cas.h"
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 10c61681b..1b501e9ae 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -5,10 +5,10 @@
#include "cacheshared.h"
#include <zencore/compactbinary.h>
-#include <zencore/stats.h>
#include <zenstore/accesstime.h>
#include <zenstore/blockstore.h>
#include <zenstore/caslog.h>
+#include <zentelemetry/stats.h>
ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h
index 1ba469431..75692cfcd 100644
--- a/src/zenstore/include/zenstore/cache/structuredcachestore.h
+++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h
@@ -4,10 +4,10 @@
#include <zencore/compactbinary.h>
#include <zencore/iohash.h>
-#include <zencore/stats.h>
#include <zenstore/cache/cache.h>
#include <zenstore/cache/cachedisklayer.h>
#include <zenstore/gc.h>
+#include <zentelemetry/stats.h>
#include <zenutil/statsreporter.h>
#include <atomic>
diff --git a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
index 2f3b6b0d7..ff4a8c3f7 100644
--- a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
+++ b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
@@ -6,9 +6,9 @@
#include <zencore/compress.h>
#include <zencore/iobuffer.h>
#include <zencore/iohash.h>
-#include <zencore/stats.h>
#include <zencore/zencore.h>
#include <zenstore/cache/cache.h>
+#include <zentelemetry/stats.h>
#include <functional>
#include <memory>
diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h
index 8918b119f..d54062476 100644
--- a/src/zenstore/include/zenstore/cidstore.h
+++ b/src/zenstore/include/zenstore/cidstore.h
@@ -5,8 +5,8 @@
#include "zenstore.h"
#include <zencore/iohash.h>
-#include <zencore/stats.h>
#include <zenstore/hashkeyset.h>
+#include <zentelemetry/stats.h>
#include <zenutil/statsreporter.h>
#include <filesystem>
diff --git a/src/zentelemetry-test/xmake.lua b/src/zentelemetry-test/xmake.lua
new file mode 100644
index 000000000..bdc60cee8
--- /dev/null
+++ b/src/zentelemetry-test/xmake.lua
@@ -0,0 +1,9 @@
+-- Copyright Epic Games, Inc. All Rights Reserved.
+
+target("zentelemetry-test")
+ set_kind("binary")
+ set_group("tests")
+ add_headerfiles("**.h")
+ add_files("*.cpp")
+ add_deps("zentelemetry")
+ add_packages("vcpkg::doctest")
diff --git a/src/zentelemetry-test/zentelemetry-test.cpp b/src/zentelemetry-test/zentelemetry-test.cpp
new file mode 100644
index 000000000..c8b067226
--- /dev/null
+++ b/src/zentelemetry-test/zentelemetry-test.cpp
@@ -0,0 +1,42 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/filesystem.h>
+#include <zencore/logging.h>
+#include <zencore/trace.h>
+#include <zentelemetry/zentelemetry.h>
+
+#include <zencore/memory/newdelete.h>
+
+#if ZEN_WITH_TESTS
+# define ZEN_TEST_WITH_RUNNER 1
+# include <zencore/testing.h>
+# include <zencore/process.h>
+#endif
+
+int
+main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
+{
+#if ZEN_WITH_TESTS
+ zen::zentelemetry_forcelinktests();
+
+# if ZEN_PLATFORM_LINUX
+ zen::IgnoreChildSignals();
+# endif
+
+# if ZEN_WITH_TRACE
+ zen::TraceInit("zenstore-test");
+ zen::TraceOptions TraceCommandlineOptions;
+ if (GetTraceOptionsFromCommandline(TraceCommandlineOptions))
+ {
+ TraceConfigure(TraceCommandlineOptions);
+ }
+# endif // ZEN_WITH_TRACE
+
+ zen::logging::InitializeLogging();
+ zen::MaximizeOpenFileCount();
+
+ return ZEN_RUN_TESTS(argc, argv);
+#else
+ return 0;
+#endif
+}
diff --git a/src/zentelemetry/include/zentelemetry/otlpencoder.h b/src/zentelemetry/include/zentelemetry/otlpencoder.h
new file mode 100644
index 000000000..ed6665781
--- /dev/null
+++ b/src/zentelemetry/include/zentelemetry/otlpencoder.h
@@ -0,0 +1,66 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/thread.h>
+#include <zentelemetry/otlptrace.h>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+
+#if ZEN_WITH_OTEL
+
+# include <protozero/pbf_builder.hpp>
+# include <protozero/types.hpp>
+
+namespace spdlog { namespace details {
+ struct log_msg;
+}} // namespace spdlog::details
+
+namespace zen::otel {
+enum class Resource : protozero::pbf_tag_type;
+}
+
+namespace zen {
+
+/**
+ * Encoder for OpenTelemetry OTLP Protobuf format
+ *
+ * Designed to encode log messages and metrics into the OTLP Protobuf format
+ * for transmission to an OpenTelemetry collector or compatible backend.
+ *
+ * Currently, only log messages and basic resource attributes are supported.
+ *
+ * Some initial support for metrics encoding is also included.
+ *
+ * Transmission is expected to be handled externally, e.g. via HTTP client.
+ *
+ */
+
+class OtlpEncoder
+{
+public:
+ OtlpEncoder();
+ ~OtlpEncoder();
+
+ void AddResourceAttribute(const std::string_view& Key, const std::string_view& Value);
+ void AddResourceAttribute(const std::string_view& Key, int64_t Value);
+
+ std::string FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const;
+ std::string FormatOtelMetrics() const;
+ std::string FormatOtelTrace(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans) const;
+
+ OtlpEncoder& operator=(const OtlpEncoder&) = delete;
+ OtlpEncoder(const OtlpEncoder&) = delete;
+
+private:
+ mutable RwLock m_ResourceLock;
+ std::unordered_map<std::string, std::string> m_ResourceAttributes;
+ std::unordered_map<std::string, int64_t> m_ResourceIntAttributes;
+
+ void AppendResourceAttributes(protozero::pbf_builder<otel::Resource>& Res) const;
+};
+
+} // namespace zen
+
+#endif
diff --git a/src/zentelemetry/include/zentelemetry/otlptrace.h b/src/zentelemetry/include/zentelemetry/otlptrace.h
new file mode 100644
index 000000000..ebecff91a
--- /dev/null
+++ b/src/zentelemetry/include/zentelemetry/otlptrace.h
@@ -0,0 +1,268 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/refcount.h>
+#include <zencore/uid.h>
+
+#include <span>
+#include <string>
+
+#define ZEN_WITH_OTEL 1
+
+#if ZEN_WITH_OTEL
+# define ZEN_CONCAT(a, b) a##b
+# define IMPL_ZEN_OTEL_SPAN(Name, Line) ::zen::otel::ScopedSpan ZEN_CONCAT(Span$, Line)(Name);
+# define ZEN_OTEL_SPAN(Name) IMPL_ZEN_OTEL_SPAN(Name, __LINE__)
+#else
+# define ZEN_OTEL_SPAN(Name)
+#endif
+
+namespace zen {
+class MemoryArena;
+}
+
+#if ZEN_WITH_OTEL
+
+namespace zen::otel {
+
+using AttributeList = std::span<std::pair<std::string, std::string>>;
+
+class Tracer;
+
+Tracer& GetTracer();
+
+// OLTP Span ID
+
+struct SpanId
+{
+ constexpr static size_t kSize = 8;
+
+ inline SpanId() : Id{0} {}
+ explicit SpanId(const std::span<const uint8_t, 8> Bytes) noexcept { std::copy(Bytes.begin(), Bytes.end(), Id); }
+ explicit SpanId(const Oid& InId) noexcept { memcpy(Id, reinterpret_cast<const uint8_t*>(InId.OidBits), sizeof Id); }
+
+ auto operator<=>(const SpanId& Rhs) const = default;
+ inline operator bool() const { return *this != SpanId(); }
+
+ static SpanId NewSpanId();
+
+ const char* GetData() const { return reinterpret_cast<const char*>(Id); }
+
+private:
+ uint8_t Id[kSize];
+};
+
+// OLTP Trace ID
+
+struct TraceId
+{
+ constexpr static size_t kSize = 16;
+
+ std::span<const uint8_t> GetBytes() const { return std::span<const uint8_t>(Id, kSize); }
+
+ inline TraceId() noexcept : Id{0} {}
+ explicit TraceId(const std::span<const uint8_t, kSize> Bytes) noexcept { std::copy(Bytes.begin(), Bytes.end(), Id); }
+
+ auto operator<=>(const TraceId& Rhs) const = default;
+ inline operator bool() const { return *this != TraceId(); }
+
+ static TraceId NewTraceId();
+
+ inline const char* GetData() const { return reinterpret_cast<const char*>(Id); }
+
+private:
+ uint8_t Id[kSize];
+};
+
+struct AttributePair
+{
+ const char* Key = nullptr;
+ union
+ {
+ const char* StringValue;
+ uint64_t NumericValue;
+ } Value;
+
+ uint64_t Flags = 0;
+
+ enum
+ {
+ kFlags_IsNumeric = 1 << 0,
+ kFlags_IsString = 1 << 1
+ };
+
+ bool IsNumeric() const { return (Flags & kFlags_IsNumeric) != 0; }
+ uint64_t GetNumericValue() const { return Value.NumericValue; }
+ void SetNumericValue(uint64_t InValue)
+ {
+ Value.NumericValue = InValue;
+ Flags |= kFlags_IsNumeric;
+ }
+
+ bool IsString() const { return (Flags & kFlags_IsString) != 0; }
+ const char* GetStringValue() const { return Value.StringValue; }
+ void SetStringValue(const char* InValue)
+ {
+ Value.StringValue = InValue;
+ Flags |= kFlags_IsString;
+ }
+
+ AttributePair* Next = nullptr;
+};
+
+struct Event
+{
+ Event* NextEvent = nullptr;
+ uint64_t Timestamp = 0;
+ const char* Name = nullptr;
+ AttributePair* Attributes = nullptr;
+};
+
+/** Span within a trace
+ *
+ * A span represents a single operation within a trace. Spans can be nested
+ * to form a trace tree.
+ *
+ */
+
+struct Span final : public TRefCounted<Span>
+{
+ Span& operator=(const Span&) = delete;
+ Span(const Span&) = delete;
+
+ void AddEvent(std::string_view Name);
+ void AddEvent(std::string_view Name, uint64_t Timestamp);
+ void AddEvent(std::string_view Name, const AttributeList& Attributes);
+
+ void AddAttribute(std::string_view Key, std::string_view Value);
+ void AddAttribute(std::string_view Key, uint64_t Value);
+ void AddAttributes(const AttributeList& Attributes);
+
+ void End();
+
+ enum class Kind : uint8_t // This must match otel::SpanKind
+ {
+ kInternal = 1,
+ kServer = 2,
+ kClient = 3,
+ kProducer = 4,
+ kConsumer = 5
+ };
+
+ Kind GetKind() const { return m_Kind; }
+ void SetKind(Kind InKind) { m_Kind = InKind; }
+
+ SpanId GetSpanId() const { return m_SpanId; }
+ const char* GetName() const { return m_Name; }
+ uint64_t GetStartTime() const { return m_StartTime; }
+ uint64_t GetEndTime() const { return m_EndTime; }
+ const Span* GetParentSpan() const { return m_ParentSpan; }
+ const AttributePair* GetAttributes() const { return m_Attributes; }
+ const Event* GetEvents() const { return m_Events; }
+
+ // This is used to get the current span/trace ID for logging purposes
+ static SpanId GetCurrentSpanId(TraceId& OutTraceId);
+ static Span* GetCurrentSpan();
+
+protected:
+ void* operator new(size_t Size) = delete;
+ void* operator new(size_t Size, MemoryArena& Arena);
+ void operator delete(void* Ptr) = delete;
+ void operator delete(void* Ptr, MemoryArena& Arena);
+
+ friend class ScopedSpan;
+ friend class Tracer;
+ friend class TRefCounted<Span>;
+
+ void DeleteThis() noexcept { End(); }
+
+private:
+ // Note: for now there's no locking on the span. To support spans across
+ // threads, we'll need to add locking around event addition and ending
+ // and linking of child spans etc
+
+ MemoryArena& m_Arena;
+ Span* m_NextSibling = nullptr;
+ Span* m_FirstChild = nullptr;
+ Span* m_ParentSpan = nullptr;
+ Span* m_SpanChain = nullptr;
+ Event* m_Events = nullptr;
+ AttributePair* m_Attributes = nullptr;
+ SpanId m_SpanId;
+ const char* m_Name = nullptr;
+ uint64_t m_StartTime = 0;
+ uint64_t m_EndTime = 0;
+ Kind m_Kind = Kind::kInternal;
+ bool m_Ended = false;
+
+ Span(MemoryArena& Arena, std::string_view Name, Span* SpanChain);
+ ~Span();
+};
+
+/** Scoped span helper
+ *
+ * Automatically ends the span when it goes out of scope
+ */
+
+class ScopedSpan final
+{
+public:
+ ScopedSpan(std::string_view Name);
+ ScopedSpan() = delete;
+ ~ScopedSpan();
+
+ ScopedSpan& operator=(ScopedSpan&& Rhs) = default;
+ ScopedSpan& operator=(const ScopedSpan& Rhs) = default;
+ ScopedSpan(ScopedSpan&& Rhs) = default;
+ ScopedSpan(const ScopedSpan& Rhs) = default;
+
+ Span* operator->() const { return m_Span.Get(); }
+
+private:
+ ScopedSpan(Span* InSpan, Tracer* InTracer);
+
+ Ref<Tracer> m_Tracer; // This needs to precede the span ref to ensure proper destruction order
+ Ref<Span> m_Span;
+
+ friend class Tracer;
+};
+
+/**
+ * Tracer for creating spans and managing trace state.
+ *
+ * This represents a single trace context identified by a trace ID,
+ * and is at the root of all spans created within that trace.
+ *
+ */
+
+class Tracer final : public RefCounted
+{
+public:
+ static ScopedSpan CreateSpan(std::string_view Name);
+
+private:
+ struct Impl;
+ Impl* m_Impl;
+
+ Tracer();
+ ~Tracer();
+
+ static Tracer* GetTracer();
+
+ friend class ScopedSpan;
+};
+
+class TraceRecorder : public RefCounted
+{
+public:
+ virtual void RecordSpans(TraceId Trace, std::span<const Span*> Spans) = 0;
+};
+
+void SetTraceRecorder(Ref<TraceRecorder> Recorder);
+bool IsRecording();
+
+void otlptrace_forcelink();
+
+} // namespace zen::otel
+#endif
diff --git a/src/zencore/include/zencore/stats.h b/src/zentelemetry/include/zentelemetry/stats.h
index f232cf2f4..3e67bac1c 100644
--- a/src/zencore/include/zencore/stats.h
+++ b/src/zentelemetry/include/zentelemetry/stats.h
@@ -2,7 +2,7 @@
#pragma once
-#include "zencore.h"
+#include "zentelemetry.h"
#include <zenbase/concepts.h>
diff --git a/src/zentelemetry/include/zentelemetry/zentelemetry.h b/src/zentelemetry/include/zentelemetry/zentelemetry.h
new file mode 100644
index 000000000..fc811974b
--- /dev/null
+++ b/src/zentelemetry/include/zentelemetry/zentelemetry.h
@@ -0,0 +1,9 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+namespace zen {
+
+void zentelemetry_forcelinktests();
+
+}
diff --git a/src/zentelemetry/otellogprotozero.h b/src/zentelemetry/otellogprotozero.h
new file mode 100644
index 000000000..1e2e85d2d
--- /dev/null
+++ b/src/zentelemetry/otellogprotozero.h
@@ -0,0 +1,287 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "otelprotozero.h"
+
+//////////////////////////////////////////////////////////////////////////
+//
+// OTEL .proto definitions, for reference
+//
+
+#if 0
+
+// clang-format off
+
+/////////////////////////////////////////////////////////////////////////////////////
+// logs/v1
+
+// LogsData represents the logs data that can be stored in a persistent storage,
+// OR can be embedded by other protocols that transfer OTLP logs data but do not
+// implement the OTLP protocol.
+//
+// The main difference between this message and collector protocol is that
+// in this message there will not be any "control" or "metadata" specific to
+// OTLP protocol.
+//
+// When new fields are added into this message, the OTLP request MUST be updated
+// as well.
+message LogsData {
+ // An array of ResourceLogs.
+ // For data coming from a single resource this array will typically contain
+ // one element. Intermediary nodes that receive data from multiple origins
+ // typically batch the data before forwarding further and in that case this
+ // array will contain multiple elements.
+ repeated ResourceLogs resource_logs = 1;
+}
+
+// A collection of ScopeLogs from a Resource.
+message ResourceLogs {
+ reserved 1000;
+
+ // The resource for the logs in this message.
+ // If this field is not set then resource info is unknown.
+ opentelemetry.proto.resource.v1.Resource resource = 1;
+
+ // A list of ScopeLogs that originate from a resource.
+ repeated ScopeLogs scope_logs = 2;
+
+ // The Schema URL, if known. This is the identifier of the Schema that the resource data
+ // is recorded in. Notably, the last part of the URL path is the version number of the
+ // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see
+ // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url
+ // This schema_url applies to the data in the "resource" field. It does not apply
+ // to the data in the "scope_logs" field which have their own schema_url field.
+ string schema_url = 3;
+}
+
+// A collection of Logs produced by a Scope.
+message ScopeLogs {
+ // The instrumentation scope information for the logs in this message.
+ // Semantically when InstrumentationScope isn't set, it is equivalent with
+ // an empty instrumentation scope name (unknown).
+ opentelemetry.proto.common.v1.InstrumentationScope scope = 1;
+
+ // A list of log records.
+ repeated LogRecord log_records = 2;
+
+ // The Schema URL, if known. This is the identifier of the Schema that the log data
+ // is recorded in. Notably, the last part of the URL path is the version number of the
+ // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see
+ // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url
+ // This schema_url applies to all logs in the "logs" field.
+ string schema_url = 3;
+}
+
+// Possible values for LogRecord.SeverityNumber.
+enum SeverityNumber {
+ // UNSPECIFIED is the default SeverityNumber, it MUST NOT be used.
+ SEVERITY_NUMBER_UNSPECIFIED = 0;
+ SEVERITY_NUMBER_TRACE = 1;
+ SEVERITY_NUMBER_TRACE2 = 2;
+ SEVERITY_NUMBER_TRACE3 = 3;
+ SEVERITY_NUMBER_TRACE4 = 4;
+ SEVERITY_NUMBER_DEBUG = 5;
+ SEVERITY_NUMBER_DEBUG2 = 6;
+ SEVERITY_NUMBER_DEBUG3 = 7;
+ SEVERITY_NUMBER_DEBUG4 = 8;
+ SEVERITY_NUMBER_INFO = 9;
+ SEVERITY_NUMBER_INFO2 = 10;
+ SEVERITY_NUMBER_INFO3 = 11;
+ SEVERITY_NUMBER_INFO4 = 12;
+ SEVERITY_NUMBER_WARN = 13;
+ SEVERITY_NUMBER_WARN2 = 14;
+ SEVERITY_NUMBER_WARN3 = 15;
+ SEVERITY_NUMBER_WARN4 = 16;
+ SEVERITY_NUMBER_ERROR = 17;
+ SEVERITY_NUMBER_ERROR2 = 18;
+ SEVERITY_NUMBER_ERROR3 = 19;
+ SEVERITY_NUMBER_ERROR4 = 20;
+ SEVERITY_NUMBER_FATAL = 21;
+ SEVERITY_NUMBER_FATAL2 = 22;
+ SEVERITY_NUMBER_FATAL3 = 23;
+ SEVERITY_NUMBER_FATAL4 = 24;
+}
+
+// LogRecordFlags represents constants used to interpret the
+// LogRecord.flags field, which is protobuf 'fixed32' type and is to
+// be used as bit-fields. Each non-zero value defined in this enum is
+// a bit-mask. To extract the bit-field, for example, use an
+// expression like:
+//
+// (logRecord.flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK)
+//
+enum LogRecordFlags {
+ // The zero value for the enum. Should not be used for comparisons.
+ // Instead use bitwise "and" with the appropriate mask as shown above.
+ LOG_RECORD_FLAGS_DO_NOT_USE = 0;
+
+ // Bits 0-7 are used for trace flags.
+ LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 0x000000FF;
+
+ // Bits 8-31 are reserved for future use.
+}
+
+// A log record according to OpenTelemetry Log Data Model:
+// https://github.com/open-telemetry/oteps/blob/main/text/logs/0097-log-data-model.md
+message LogRecord {
+ reserved 4;
+
+ // time_unix_nano is the time when the event occurred.
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970.
+ // Value of 0 indicates unknown or missing timestamp.
+ fixed64 time_unix_nano = 1;
+
+ // Time when the event was observed by the collection system.
+ // For events that originate in OpenTelemetry (e.g. using OpenTelemetry Logging SDK)
+ // this timestamp is typically set at the generation time and is equal to Timestamp.
+ // For events originating externally and collected by OpenTelemetry (e.g. using
+ // Collector) this is the time when OpenTelemetry's code observed the event measured
+ // by the clock of the OpenTelemetry code. This field MUST be set once the event is
+ // observed by OpenTelemetry.
+ //
+ // For converting OpenTelemetry log data to formats that support only one timestamp or
+ // when receiving OpenTelemetry log data by recipients that support only one timestamp
+ // internally the following logic is recommended:
+ // - Use time_unix_nano if it is present, otherwise use observed_time_unix_nano.
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970.
+ // Value of 0 indicates unknown or missing timestamp.
+ fixed64 observed_time_unix_nano = 11;
+
+ // Numerical value of the severity, normalized to values described in Log Data Model.
+ // [Optional].
+ SeverityNumber severity_number = 2;
+
+ // The severity text (also known as log level). The original string representation as
+ // it is known at the source. [Optional].
+ string severity_text = 3;
+
+ // A value containing the body of the log record. Can be for example a human-readable
+ // string message (including multi-line) describing the event in a free form or it can
+ // be a structured data composed of arrays and maps of other values. [Optional].
+ opentelemetry.proto.common.v1.AnyValue body = 5;
+
+ // Additional attributes that describe the specific event occurrence. [Optional].
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ repeated opentelemetry.proto.common.v1.KeyValue attributes = 6;
+ uint32 dropped_attributes_count = 7;
+
+ // Flags, a bit field. 8 least significant bits are the trace flags as
+ // defined in W3C Trace Context specification. 24 most significant bits are reserved
+ // and must be set to 0. Readers must not assume that 24 most significant bits
+ // will be zero and must correctly mask the bits when reading 8-bit trace flag (use
+ // flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK). [Optional].
+ fixed32 flags = 8;
+
+ // A unique identifier for a trace. All logs from the same trace share
+ // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR
+ // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON
+ // is zero-length and thus is also invalid).
+ //
+ // This field is optional.
+ //
+ // The receivers SHOULD assume that the log record is not associated with a
+ // trace if any of the following is true:
+ // - the field is not present,
+ // - the field contains an invalid value.
+ bytes trace_id = 9;
+
+ // A unique identifier for a span within a trace, assigned when the span
+ // is created. The ID is an 8-byte array. An ID with all zeroes OR of length
+ // other than 8 bytes is considered invalid (empty string in OTLP/JSON
+ // is zero-length and thus is also invalid).
+ //
+ // This field is optional. If the sender specifies a valid span_id then it SHOULD also
+ // specify a valid trace_id.
+ //
+ // The receivers SHOULD assume that the log record is not associated with a
+ // span if any of the following is true:
+ // - the field is not present,
+ // - the field contains an invalid value.
+ bytes span_id = 10;
+
+ // A unique identifier of event category/type.
+ // All events with the same event_name are expected to conform to the same
+ // schema for both their attributes and their body.
+ //
+ // Recommended to be fully qualified and short (no longer than 256 characters).
+ //
+ // Presence of event_name on the log record identifies this record
+ // as an event.
+ //
+ // [Optional].
+ string event_name = 12;
+}
+// clang-format on
+#endif
+
+namespace zen::otel {
+
+// Possible values for LogRecord.SeverityNumber.
+enum SeverityNumber
+{
+ // UNSPECIFIED is the default SeverityNumber, it MUST NOT be used.
+ SEVERITY_NUMBER_UNSPECIFIED = 0,
+ SEVERITY_NUMBER_TRACE = 1,
+ SEVERITY_NUMBER_TRACE2 = 2,
+ SEVERITY_NUMBER_TRACE3 = 3,
+ SEVERITY_NUMBER_TRACE4 = 4,
+ SEVERITY_NUMBER_DEBUG = 5,
+ SEVERITY_NUMBER_DEBUG2 = 6,
+ SEVERITY_NUMBER_DEBUG3 = 7,
+ SEVERITY_NUMBER_DEBUG4 = 8,
+ SEVERITY_NUMBER_INFO = 9,
+ SEVERITY_NUMBER_INFO2 = 10,
+ SEVERITY_NUMBER_INFO3 = 11,
+ SEVERITY_NUMBER_INFO4 = 12,
+ SEVERITY_NUMBER_WARN = 13,
+ SEVERITY_NUMBER_WARN2 = 14,
+ SEVERITY_NUMBER_WARN3 = 15,
+ SEVERITY_NUMBER_WARN4 = 16,
+ SEVERITY_NUMBER_ERROR = 17,
+ SEVERITY_NUMBER_ERROR2 = 18,
+ SEVERITY_NUMBER_ERROR3 = 19,
+ SEVERITY_NUMBER_ERROR4 = 20,
+ SEVERITY_NUMBER_FATAL = 21,
+ SEVERITY_NUMBER_FATAL2 = 22,
+ SEVERITY_NUMBER_FATAL3 = 23,
+ SEVERITY_NUMBER_FATAL4 = 24
+};
+
+enum class LogRecord : protozero::pbf_tag_type
+{
+ required_fixed64_time_unix_nano = 1,
+ optional_SeverityNumber_severity_number = 2,
+ optional_string_severity_text = 3,
+ optional_anyvalue_body = 5,
+ optional_repeated_kv_attributes = 6,
+ optional_uint32_dropped_attributes_count = 7,
+ optional_fixed32_flags = 8,
+ optional_bytes_trace_id = 9,
+ optional_bytes_span_id = 10,
+ optional_fixed64_observed_time_unix_nano = 11,
+ optional_event_name = 12,
+};
+
+enum class ScopeLogs : protozero::pbf_tag_type
+{
+ required_InstrumentationScope_scope = 1,
+ required_repeated_LogRecord_log_records = 2,
+ optional_string_schema_url = 3
+};
+
+enum class ResourceLogs : protozero::pbf_tag_type
+{
+ optional_Resource_resource = 1,
+ required_repeated_ScopeLogs_scope_logs = 2,
+ optional_string_schema_url = 3
+};
+
+enum class LogsData : protozero::pbf_tag_type
+{
+ required_repeated_ResourceLogs_resource_logs = 1
+};
+
+} // namespace zen::otel
diff --git a/src/zentelemetry/otelmetricsprotozero.h b/src/zentelemetry/otelmetricsprotozero.h
new file mode 100644
index 000000000..12bae0d75
--- /dev/null
+++ b/src/zentelemetry/otelmetricsprotozero.h
@@ -0,0 +1,953 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "otelprotozero.h"
+
+//////////////////////////////////////////////////////////////////////////
+//
+// OTEL .proto definitions, for reference
+//
+
+#if 0
+
+// clang-format off
+
+//////////////////////////////////////////////////////////////////////////
+// metrics/v1/metrics.proto
+//
+
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package opentelemetry.proto.metrics.v1;
+
+import "opentelemetry/proto/common/v1/common.proto";
+import "opentelemetry/proto/resource/v1/resource.proto";
+
+option csharp_namespace = "OpenTelemetry.Proto.Metrics.V1";
+option java_multiple_files = true;
+option java_package = "io.opentelemetry.proto.metrics.v1";
+option java_outer_classname = "MetricsProto";
+option go_package = "go.opentelemetry.io/proto/otlp/metrics/v1";
+
+// MetricsData represents the metrics data that can be stored in a persistent
+// storage, OR can be embedded by other protocols that transfer OTLP metrics
+// data but do not implement the OTLP protocol.
+//
+// MetricsData
+// └─── ResourceMetrics
+// ├── Resource
+// ├── SchemaURL
+// └── ScopeMetrics
+// ├── Scope
+// ├── SchemaURL
+// └── Metric
+// ├── Name
+// ├── Description
+// ├── Unit
+// └── data
+// ├── Gauge
+// ├── Sum
+// ├── Histogram
+// ├── ExponentialHistogram
+// └── Summary
+//
+// The main difference between this message and collector protocol is that
+// in this message there will not be any "control" or "metadata" specific to
+// OTLP protocol.
+//
+// When new fields are added into this message, the OTLP request MUST be updated
+// as well.
+message MetricsData {
+ // An array of ResourceMetrics.
+ // For data coming from a single resource this array will typically contain
+ // one element. Intermediary nodes that receive data from multiple origins
+ // typically batch the data before forwarding further and in that case this
+ // array will contain multiple elements.
+ repeated ResourceMetrics resource_metrics = 1;
+}
+
+// A collection of ScopeMetrics from a Resource.
+message ResourceMetrics {
+ reserved 1000;
+
+ // The resource for the metrics in this message.
+ // If this field is not set then no resource info is known.
+ opentelemetry.proto.resource.v1.Resource resource = 1;
+
+ // A list of metrics that originate from a resource.
+ repeated ScopeMetrics scope_metrics = 2;
+
+ // The Schema URL, if known. This is the identifier of the Schema that the resource data
+ // is recorded in. Notably, the last part of the URL path is the version number of the
+ // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see
+ // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url
+ // This schema_url applies to the data in the "resource" field. It does not apply
+ // to the data in the "scope_metrics" field which have their own schema_url field.
+ string schema_url = 3;
+}
+
+// A collection of Metrics produced by an Scope.
+message ScopeMetrics {
+ // The instrumentation scope information for the metrics in this message.
+ // Semantically when InstrumentationScope isn't set, it is equivalent with
+ // an empty instrumentation scope name (unknown).
+ opentelemetry.proto.common.v1.InstrumentationScope scope = 1;
+
+ // A list of metrics that originate from an instrumentation library.
+ repeated Metric metrics = 2;
+
+ // The Schema URL, if known. This is the identifier of the Schema that the metric data
+ // is recorded in. Notably, the last part of the URL path is the version number of the
+ // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see
+ // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url
+ // This schema_url applies to all metrics in the "metrics" field.
+ string schema_url = 3;
+}
+
+// Defines a Metric which has one or more timeseries. The following is a
+// brief summary of the Metric data model. For more details, see:
+//
+// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md
+//
+// The data model and relation between entities is shown in the
+// diagram below. Here, "DataPoint" is the term used to refer to any
+// one of the specific data point value types, and "points" is the term used
+// to refer to any one of the lists of points contained in the Metric.
+//
+// - Metric is composed of a metadata and data.
+// - Metadata part contains a name, description, unit.
+// - Data is one of the possible types (Sum, Gauge, Histogram, Summary).
+// - DataPoint contains timestamps, attributes, and one of the possible value type
+// fields.
+//
+// Metric
+// +------------+
+// |name |
+// |description |
+// |unit | +------------------------------------+
+// |data |---> |Gauge, Sum, Histogram, Summary, ... |
+// +------------+ +------------------------------------+
+//
+// Data [One of Gauge, Sum, Histogram, Summary, ...]
+// +-----------+
+// |... | // Metadata about the Data.
+// |points |--+
+// +-----------+ |
+// | +---------------------------+
+// | |DataPoint 1 |
+// v |+------+------+ +------+ |
+// +-----+ ||label |label |...|label | |
+// | 1 |-->||value1|value2|...|valueN| |
+// +-----+ |+------+------+ +------+ |
+// | . | |+-----+ |
+// | . | ||value| |
+// | . | |+-----+ |
+// | . | +---------------------------+
+// | . | .
+// | . | .
+// | . | .
+// | . | +---------------------------+
+// | . | |DataPoint M |
+// +-----+ |+------+------+ +------+ |
+// | M |-->||label |label |...|label | |
+// +-----+ ||value1|value2|...|valueN| |
+// |+------+------+ +------+ |
+// |+-----+ |
+// ||value| |
+// |+-----+ |
+// +---------------------------+
+//
+// Each distinct type of DataPoint represents the output of a specific
+// aggregation function, the result of applying the DataPoint's
+// associated function of to one or more measurements.
+//
+// All DataPoint types have three common fields:
+// - Attributes includes key-value pairs associated with the data point
+// - TimeUnixNano is required, set to the end time of the aggregation
+// - StartTimeUnixNano is optional, but strongly encouraged for DataPoints
+// having an AggregationTemporality field, as discussed below.
+//
+// Both TimeUnixNano and StartTimeUnixNano values are expressed as
+// UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970.
+//
+// # TimeUnixNano
+//
+// This field is required, having consistent interpretation across
+// DataPoint types. TimeUnixNano is the moment corresponding to when
+// the data point's aggregate value was captured.
+//
+// Data points with the 0 value for TimeUnixNano SHOULD be rejected
+// by consumers.
+//
+// # StartTimeUnixNano
+//
+// StartTimeUnixNano in general allows detecting when a sequence of
+// observations is unbroken. This field indicates to consumers the
+// start time for points with cumulative and delta
+// AggregationTemporality, and it should be included whenever possible
+// to support correct rate calculation. Although it may be omitted
+// when the start time is truly unknown, setting StartTimeUnixNano is
+// strongly encouraged.
+message Metric {
+ reserved 4, 6, 8;
+
+ // The name of the metric.
+ string name = 1;
+
+ // A description of the metric, which can be used in documentation.
+ string description = 2;
+
+ // The unit in which the metric value is reported. Follows the format
+ // described by https://unitsofmeasure.org/ucum.html.
+ string unit = 3;
+
+ // Data determines the aggregation type (if any) of the metric, what is the
+ // reported value type for the data points, as well as the relatationship to
+ // the time interval over which they are reported.
+ oneof data {
+ Gauge gauge = 5;
+ Sum sum = 7;
+ Histogram histogram = 9;
+ ExponentialHistogram exponential_histogram = 10;
+ Summary summary = 11;
+ }
+
+ // Additional metadata attributes that describe the metric. [Optional].
+ // Attributes are non-identifying.
+ // Consumers SHOULD NOT need to be aware of these attributes.
+ // These attributes MAY be used to encode information allowing
+ // for lossless roundtrip translation to / from another data model.
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ repeated opentelemetry.proto.common.v1.KeyValue metadata = 12;
+}
+
+// Gauge represents the type of a scalar metric that always exports the
+// "current value" for every data point. It should be used for an "unknown"
+// aggregation.
+//
+// A Gauge does not support different aggregation temporalities. Given the
+// aggregation is unknown, points cannot be combined using the same
+// aggregation, regardless of aggregation temporalities. Therefore,
+// AggregationTemporality is not included. Consequently, this also means
+// "StartTimeUnixNano" is ignored for all data points.
+message Gauge {
+ // The time series data points.
+ // Note: Multiple time series may be included (same timestamp, different attributes).
+ repeated NumberDataPoint data_points = 1;
+}
+
+// Sum represents the type of a scalar metric that is calculated as a sum of all
+// reported measurements over a time interval.
+message Sum {
+ // The time series data points.
+ // Note: Multiple time series may be included (same timestamp, different attributes).
+ repeated NumberDataPoint data_points = 1;
+
+ // aggregation_temporality describes if the aggregator reports delta changes
+ // since last report time, or cumulative changes since a fixed start time.
+ AggregationTemporality aggregation_temporality = 2;
+
+ // Represents whether the sum is monotonic.
+ bool is_monotonic = 3;
+}
+
+// Histogram represents the type of a metric that is calculated by aggregating
+// as a Histogram of all reported measurements over a time interval.
+message Histogram {
+ // The time series data points.
+ // Note: Multiple time series may be included (same timestamp, different attributes).
+ repeated HistogramDataPoint data_points = 1;
+
+ // aggregation_temporality describes if the aggregator reports delta changes
+ // since last report time, or cumulative changes since a fixed start time.
+ AggregationTemporality aggregation_temporality = 2;
+}
+
+// ExponentialHistogram represents the type of a metric that is calculated by aggregating
+// as a ExponentialHistogram of all reported double measurements over a time interval.
+message ExponentialHistogram {
+ // The time series data points.
+ // Note: Multiple time series may be included (same timestamp, different attributes).
+ repeated ExponentialHistogramDataPoint data_points = 1;
+
+ // aggregation_temporality describes if the aggregator reports delta changes
+ // since last report time, or cumulative changes since a fixed start time.
+ AggregationTemporality aggregation_temporality = 2;
+}
+
+// Summary metric data are used to convey quantile summaries,
+// a Prometheus (see: https://prometheus.io/docs/concepts/metric_types/#summary)
+// and OpenMetrics (see: https://github.com/prometheus/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45)
+// data type. These data points cannot always be merged in a meaningful way.
+// While they can be useful in some applications, histogram data points are
+// recommended for new applications.
+// Summary metrics do not have an aggregation temporality field. This is
+// because the count and sum fields of a SummaryDataPoint are assumed to be
+// cumulative values.
+message Summary {
+ // The time series data points.
+ // Note: Multiple time series may be included (same timestamp, different attributes).
+ repeated SummaryDataPoint data_points = 1;
+}
+
+// AggregationTemporality defines how a metric aggregator reports aggregated
+// values. It describes how those values relate to the time interval over
+// which they are aggregated.
+enum AggregationTemporality {
+ // UNSPECIFIED is the default AggregationTemporality, it MUST not be used.
+ AGGREGATION_TEMPORALITY_UNSPECIFIED = 0;
+
+ // DELTA is an AggregationTemporality for a metric aggregator which reports
+ // changes since last report time. Successive metrics contain aggregation of
+ // values from continuous and non-overlapping intervals.
+ //
+ // The values for a DELTA metric are based only on the time interval
+ // associated with one measurement cycle. There is no dependency on
+ // previous measurements like is the case for CUMULATIVE metrics.
+ //
+ // For example, consider a system measuring the number of requests that
+ // it receives and reports the sum of these requests every second as a
+ // DELTA metric:
+ //
+ // 1. The system starts receiving at time=t_0.
+ // 2. A request is received, the system measures 1 request.
+ // 3. A request is received, the system measures 1 request.
+ // 4. A request is received, the system measures 1 request.
+ // 5. The 1 second collection cycle ends. A metric is exported for the
+ // number of requests received over the interval of time t_0 to
+ // t_0+1 with a value of 3.
+ // 6. A request is received, the system measures 1 request.
+ // 7. A request is received, the system measures 1 request.
+ // 8. The 1 second collection cycle ends. A metric is exported for the
+ // number of requests received over the interval of time t_0+1 to
+ // t_0+2 with a value of 2.
+ AGGREGATION_TEMPORALITY_DELTA = 1;
+
+ // CUMULATIVE is an AggregationTemporality for a metric aggregator which
+ // reports changes since a fixed start time. This means that current values
+ // of a CUMULATIVE metric depend on all previous measurements since the
+ // start time. Because of this, the sender is required to retain this state
+ // in some form. If this state is lost or invalidated, the CUMULATIVE metric
+ // values MUST be reset and a new fixed start time following the last
+ // reported measurement time sent MUST be used.
+ //
+ // For example, consider a system measuring the number of requests that
+ // it receives and reports the sum of these requests every second as a
+ // CUMULATIVE metric:
+ //
+ // 1. The system starts receiving at time=t_0.
+ // 2. A request is received, the system measures 1 request.
+ // 3. A request is received, the system measures 1 request.
+ // 4. A request is received, the system measures 1 request.
+ // 5. The 1 second collection cycle ends. A metric is exported for the
+ // number of requests received over the interval of time t_0 to
+ // t_0+1 with a value of 3.
+ // 6. A request is received, the system measures 1 request.
+ // 7. A request is received, the system measures 1 request.
+ // 8. The 1 second collection cycle ends. A metric is exported for the
+ // number of requests received over the interval of time t_0 to
+ // t_0+2 with a value of 5.
+ // 9. The system experiences a fault and loses state.
+ // 10. The system recovers and resumes receiving at time=t_1.
+ // 11. A request is received, the system measures 1 request.
+ // 12. The 1 second collection cycle ends. A metric is exported for the
+ // number of requests received over the interval of time t_1 to
+ // t_0+1 with a value of 1.
+ //
+ // Note: Even though, when reporting changes since last report time, using
+ // CUMULATIVE is valid, it is not recommended. This may cause problems for
+ // systems that do not use start_time to determine when the aggregation
+ // value was reset (e.g. Prometheus).
+ AGGREGATION_TEMPORALITY_CUMULATIVE = 2;
+}
+
+// DataPointFlags is defined as a protobuf 'uint32' type and is to be used as a
+// bit-field representing 32 distinct boolean flags. Each flag defined in this
+// enum is a bit-mask. To test the presence of a single flag in the flags of
+// a data point, for example, use an expression like:
+//
+// (point.flags & DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) == DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK
+//
+enum DataPointFlags {
+ // The zero value for the enum. Should not be used for comparisons.
+ // Instead use bitwise "and" with the appropriate mask as shown above.
+ DATA_POINT_FLAGS_DO_NOT_USE = 0;
+
+ // This DataPoint is valid but has no recorded value. This value
+ // SHOULD be used to reflect explicitly missing data in a series, as
+ // for an equivalent to the Prometheus "staleness marker".
+ DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK = 1;
+
+ // Bits 2-31 are reserved for future use.
+}
+
+// NumberDataPoint is a single data point in a timeseries that describes the
+// time-varying scalar value of a metric.
+message NumberDataPoint {
+ reserved 1;
+
+ // The set of key/value pairs that uniquely identify the timeseries from
+ // where this point belongs. The list may be empty (may contain 0 elements).
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ //
+ // The attribute values SHOULD NOT contain empty values.
+ // The attribute values SHOULD NOT contain bytes values.
+ // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+ // double values.
+ // The attribute values SHOULD NOT contain kvlist values.
+ // The behavior of software that receives attributes containing such values can be unpredictable.
+ // These restrictions can change in a minor release.
+ // The restrictions take origin from the OpenTelemetry specification:
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+ repeated opentelemetry.proto.common.v1.KeyValue attributes = 7;
+
+ // StartTimeUnixNano is optional but strongly encouraged, see the
+ // the detailed comments above Metric.
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January
+ // 1970.
+ fixed64 start_time_unix_nano = 2;
+
+ // TimeUnixNano is required, see the detailed comments above Metric.
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January
+ // 1970.
+ fixed64 time_unix_nano = 3;
+
+ // The value itself. A point is considered invalid when one of the recognized
+ // value fields is not present inside this oneof.
+ oneof value {
+ double as_double = 4;
+ sfixed64 as_int = 6;
+ }
+
+ // (Optional) List of exemplars collected from
+ // measurements that were used to form the data point
+ repeated Exemplar exemplars = 5;
+
+ // Flags that apply to this specific data point. See DataPointFlags
+ // for the available flags and their meaning.
+ uint32 flags = 8;
+}
+
+// HistogramDataPoint is a single data point in a timeseries that describes the
+// time-varying values of a Histogram. A Histogram contains summary statistics
+// for a population of values, it may optionally contain the distribution of
+// those values across a set of buckets.
+//
+// If the histogram contains the distribution of values, then both
+// "explicit_bounds" and "bucket counts" fields must be defined.
+// If the histogram does not contain the distribution of values, then both
+// "explicit_bounds" and "bucket_counts" must be omitted and only "count" and
+// "sum" are known.
+message HistogramDataPoint {
+ reserved 1;
+
+ // The set of key/value pairs that uniquely identify the timeseries from
+ // where this point belongs. The list may be empty (may contain 0 elements).
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ //
+ // The attribute values SHOULD NOT contain empty values.
+ // The attribute values SHOULD NOT contain bytes values.
+ // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+ // double values.
+ // The attribute values SHOULD NOT contain kvlist values.
+ // The behavior of software that receives attributes containing such values can be unpredictable.
+ // These restrictions can change in a minor release.
+ // The restrictions take origin from the OpenTelemetry specification:
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+ repeated opentelemetry.proto.common.v1.KeyValue attributes = 9;
+
+ // StartTimeUnixNano is optional but strongly encouraged, see the
+ // the detailed comments above Metric.
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January
+ // 1970.
+ fixed64 start_time_unix_nano = 2;
+
+ // TimeUnixNano is required, see the detailed comments above Metric.
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January
+ // 1970.
+ fixed64 time_unix_nano = 3;
+
+ // count is the number of values in the population. Must be non-negative. This
+ // value must be equal to the sum of the "count" fields in buckets if a
+ // histogram is provided.
+ fixed64 count = 4;
+
+ // sum of the values in the population. If count is zero then this field
+ // must be zero.
+ //
+ // Note: Sum should only be filled out when measuring non-negative discrete
+ // events, and is assumed to be monotonic over the values of these events.
+ // Negative events *can* be recorded, but sum should not be filled out when
+ // doing so. This is specifically to enforce compatibility w/ OpenMetrics,
+ // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram
+ optional double sum = 5;
+
+ // bucket_counts is an optional field contains the count values of histogram
+ // for each bucket.
+ //
+ // The sum of the bucket_counts must equal the value in the count field.
+ //
+ // The number of elements in bucket_counts array must be by one greater than
+ // the number of elements in explicit_bounds array. The exception to this rule
+ // is when the length of bucket_counts is 0, then the length of explicit_bounds
+ // must also be 0.
+ repeated fixed64 bucket_counts = 6;
+
+ // explicit_bounds specifies buckets with explicitly defined bounds for values.
+ //
+ // The boundaries for bucket at index i are:
+ //
+ // (-infinity, explicit_bounds[i]] for i == 0
+ // (explicit_bounds[i-1], explicit_bounds[i]] for 0 < i < size(explicit_bounds)
+ // (explicit_bounds[i-1], +infinity) for i == size(explicit_bounds)
+ //
+ // The values in the explicit_bounds array must be strictly increasing.
+ //
+ // Histogram buckets are inclusive of their upper boundary, except the last
+ // bucket where the boundary is at infinity. This format is intentionally
+ // compatible with the OpenMetrics histogram definition.
+ //
+ // If bucket_counts length is 0 then explicit_bounds length must also be 0,
+ // otherwise the data point is invalid.
+ repeated double explicit_bounds = 7;
+
+ // (Optional) List of exemplars collected from
+ // measurements that were used to form the data point
+ repeated Exemplar exemplars = 8;
+
+ // Flags that apply to this specific data point. See DataPointFlags
+ // for the available flags and their meaning.
+ uint32 flags = 10;
+
+ // min is the minimum value over (start_time, end_time].
+ optional double min = 11;
+
+ // max is the maximum value over (start_time, end_time].
+ optional double max = 12;
+}
+
+// ExponentialHistogramDataPoint is a single data point in a timeseries that describes the
+// time-varying values of a ExponentialHistogram of double values. A ExponentialHistogram contains
+// summary statistics for a population of values, it may optionally contain the
+// distribution of those values across a set of buckets.
+//
+message ExponentialHistogramDataPoint {
+ // The set of key/value pairs that uniquely identify the timeseries from
+ // where this point belongs. The list may be empty (may contain 0 elements).
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ //
+ // The attribute values SHOULD NOT contain empty values.
+ // The attribute values SHOULD NOT contain bytes values.
+ // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+ // double values.
+ // The attribute values SHOULD NOT contain kvlist values.
+ // The behavior of software that receives attributes containing such values can be unpredictable.
+ // These restrictions can change in a minor release.
+ // The restrictions take origin from the OpenTelemetry specification:
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+ repeated opentelemetry.proto.common.v1.KeyValue attributes = 1;
+
+ // StartTimeUnixNano is optional but strongly encouraged, see the
+ // the detailed comments above Metric.
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January
+ // 1970.
+ fixed64 start_time_unix_nano = 2;
+
+ // TimeUnixNano is required, see the detailed comments above Metric.
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January
+ // 1970.
+ fixed64 time_unix_nano = 3;
+
+ // The number of values in the population. Must be
+ // non-negative. This value must be equal to the sum of the "bucket_counts"
+ // values in the positive and negative Buckets plus the "zero_count" field.
+ fixed64 count = 4;
+
+ // The sum of the values in the population. If count is zero then this field
+ // must be zero.
+ //
+ // Note: Sum should only be filled out when measuring non-negative discrete
+ // events, and is assumed to be monotonic over the values of these events.
+ // Negative events *can* be recorded, but sum should not be filled out when
+ // doing so. This is specifically to enforce compatibility w/ OpenMetrics,
+ // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram
+ optional double sum = 5;
+
+ // scale describes the resolution of the histogram. Boundaries are
+ // located at powers of the base, where:
+ //
+ // base = (2^(2^-scale))
+ //
+ // The histogram bucket identified by `index`, a signed integer,
+ // contains values that are greater than (base^index) and
+ // less than or equal to (base^(index+1)).
+ //
+ // The positive and negative ranges of the histogram are expressed
+ // separately. Negative values are mapped by their absolute value
+ // into the negative range using the same scale as the positive range.
+ //
+ // scale is not restricted by the protocol, as the permissible
+ // values depend on the range of the data.
+ sint32 scale = 6;
+
+ // The count of values that are either exactly zero or
+ // within the region considered zero by the instrumentation at the
+ // tolerated degree of precision. This bucket stores values that
+ // cannot be expressed using the standard exponential formula as
+ // well as values that have been rounded to zero.
+ //
+ // Implementations MAY consider the zero bucket to have probability
+ // mass equal to (zero_count / count).
+ fixed64 zero_count = 7;
+
+ // positive carries the positive range of exponential bucket counts.
+ Buckets positive = 8;
+
+ // negative carries the negative range of exponential bucket counts.
+ Buckets negative = 9;
+
+ // Buckets are a set of bucket counts, encoded in a contiguous array
+ // of counts.
+ message Buckets {
+ // The bucket index of the first entry in the bucket_counts array.
+ //
+ // Note: This uses a varint encoding as a simple form of compression.
+ sint32 offset = 1;
+
+ // An array of count values, where bucket_counts[i] carries
+ // the count of the bucket at index (offset+i). bucket_counts[i] is the count
+ // of values greater than base^(offset+i) and less than or equal to
+ // base^(offset+i+1).
+ //
+ // Note: By contrast, the explicit HistogramDataPoint uses
+ // fixed64. This field is expected to have many buckets,
+ // especially zeros, so uint64 has been selected to ensure
+ // varint encoding.
+ repeated uint64 bucket_counts = 2;
+ }
+
+ // Flags that apply to this specific data point. See DataPointFlags
+ // for the available flags and their meaning.
+ uint32 flags = 10;
+
+ // (Optional) List of exemplars collected from
+ // measurements that were used to form the data point
+ repeated Exemplar exemplars = 11;
+
+ // The minimum value over (start_time, end_time].
+ optional double min = 12;
+
+ // The maximum value over (start_time, end_time].
+ optional double max = 13;
+
+ // ZeroThreshold may be optionally set to convey the width of the zero
+ // region. Where the zero region is defined as the closed interval
+ // [-ZeroThreshold, ZeroThreshold].
+ // When ZeroThreshold is 0, zero count bucket stores values that cannot be
+ // expressed using the standard exponential formula as well as values that
+ // have been rounded to zero.
+ double zero_threshold = 14;
+}
+
+// SummaryDataPoint is a single data point in a timeseries that describes the
+// time-varying values of a Summary metric. The count and sum fields represent
+// cumulative values.
+message SummaryDataPoint {
+ reserved 1;
+
+ // The set of key/value pairs that uniquely identify the timeseries from
+ // where this point belongs. The list may be empty (may contain 0 elements).
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ //
+ // The attribute values SHOULD NOT contain empty values.
+ // The attribute values SHOULD NOT contain bytes values.
+ // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+ // double values.
+ // The attribute values SHOULD NOT contain kvlist values.
+ // The behavior of software that receives attributes containing such values can be unpredictable.
+ // These restrictions can change in a minor release.
+ // The restrictions take origin from the OpenTelemetry specification:
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+ repeated opentelemetry.proto.common.v1.KeyValue attributes = 7;
+
+ // StartTimeUnixNano is optional but strongly encouraged, see the
+ // the detailed comments above Metric.
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January
+ // 1970.
+ fixed64 start_time_unix_nano = 2;
+
+ // TimeUnixNano is required, see the detailed comments above Metric.
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January
+ // 1970.
+ fixed64 time_unix_nano = 3;
+
+ // count is the number of values in the population. Must be non-negative.
+ fixed64 count = 4;
+
+ // sum of the values in the population. If count is zero then this field
+ // must be zero.
+ //
+ // Note: Sum should only be filled out when measuring non-negative discrete
+ // events, and is assumed to be monotonic over the values of these events.
+ // Negative events *can* be recorded, but sum should not be filled out when
+ // doing so. This is specifically to enforce compatibility w/ OpenMetrics,
+ // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#summary
+ double sum = 5;
+
+ // Represents the value at a given quantile of a distribution.
+ //
+ // To record Min and Max values following conventions are used:
+ // - The 1.0 quantile is equivalent to the maximum value observed.
+ // - The 0.0 quantile is equivalent to the minimum value observed.
+ //
+ // See the following issue for more context:
+ // https://github.com/open-telemetry/opentelemetry-proto/issues/125
+ message ValueAtQuantile {
+ // The quantile of a distribution. Must be in the interval
+ // [0.0, 1.0].
+ double quantile = 1;
+
+ // The value at the given quantile of a distribution.
+ //
+ // Quantile values must NOT be negative.
+ double value = 2;
+ }
+
+ // (Optional) list of values at different quantiles of the distribution calculated
+ // from the current snapshot. The quantiles must be strictly increasing.
+ repeated ValueAtQuantile quantile_values = 6;
+
+ // Flags that apply to this specific data point. See DataPointFlags
+ // for the available flags and their meaning.
+ uint32 flags = 8;
+}
+
+// A representation of an exemplar, which is a sample input measurement.
+// Exemplars also hold information about the environment when the measurement
+// was recorded, for example the span and trace ID of the active span when the
+// exemplar was recorded.
+message Exemplar {
+ reserved 1;
+
+ // The set of key/value pairs that were filtered out by the aggregator, but
+ // recorded alongside the original measurement. Only key/value pairs that were
+ // filtered out by the aggregator should be included
+ repeated opentelemetry.proto.common.v1.KeyValue filtered_attributes = 7;
+
+ // time_unix_nano is the exact time when this exemplar was recorded
+ //
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January
+ // 1970.
+ fixed64 time_unix_nano = 2;
+
+ // The value of the measurement that was recorded. An exemplar is
+ // considered invalid when one of the recognized value fields is not present
+ // inside this oneof.
+ oneof value {
+ double as_double = 3;
+ sfixed64 as_int = 6;
+ }
+
+ // (Optional) Span ID of the exemplar trace.
+ // span_id may be missing if the measurement is not recorded inside a trace
+ // or if the trace is not sampled.
+ bytes span_id = 4;
+
+ // (Optional) Trace ID of the exemplar trace.
+ // trace_id may be missing if the measurement is not recorded inside a trace
+ // or if the trace is not sampled.
+ bytes trace_id = 5;
+}
+
+// clang-format on
+#endif
+
+namespace zen::otel {
+
+// Metrics
+
+enum class MetricsData : protozero::pbf_tag_type
+{
+ repeated_ResourceMetrics_resource_metrics = 1
+};
+
+enum class ResourceMetrics : protozero::pbf_tag_type
+{
+ Resource_resource = 1,
+ repeated_ScopeMetrics_scope_metrics = 2,
+ string_schema_url = 3
+};
+
+// A collection of Metrics produced by an Scope.
+enum class ScopeMetrics : protozero::pbf_tag_type
+{
+ InstrumentationScope_scope = 1,
+ repeated_Metric_metrics = 2,
+ string_schema_url = 3
+};
+
+enum class Metric : protozero::pbf_tag_type
+{
+ string_name = 1,
+ string_description = 2,
+ string_unit = 3,
+
+ oneof_data_Gauge_gauge = 5,
+ oneof_data_Sum_sum = 7,
+ oneof_data_Histogram_histogram = 9,
+ oneof_data_ExponentialHistogram_exponential_histogram = 10,
+ oneof_data_Summary_summary = 11,
+
+ repeated_KeyValue_metadata = 12
+};
+
+enum class Gauge : protozero::pbf_tag_type
+{
+ repeated_NumberDataPoint_data_points = 1
+};
+
+enum class Sum : protozero::pbf_tag_type
+{
+ repeated_NumberDataPoint_data_points = 1,
+ AggregationTemporality_aggregation_temporality = 2,
+ bool_is_monotonic = 3
+};
+
+enum class Histogram : protozero::pbf_tag_type
+{
+ repeated_HistogramDataPoint_data_points = 1,
+ AggregationTemporality_aggregation_temporality = 2
+};
+
+enum class ExponentialHistogram : protozero::pbf_tag_type
+{
+ repeated_ExponentialHistogramDataPoint_data_points = 1,
+ AggregationTemporality_aggregation_temporality = 2
+};
+
+enum class Summary : protozero::pbf_tag_type
+{
+ repeated_SummaryDataPoint_data_points = 1
+};
+
+enum AggregationTemporality
+{
+ AGGREGATION_TEMPORALITY_UNSPECIFIED = 0,
+ AGGREGATION_TEMPORALITY_DELTA = 1,
+ AGGREGATION_TEMPORALITY_CUMULATIVE = 2
+};
+
+enum DataPointFlags
+{
+ DATA_POINT_FLAGS_DO_NOT_USE = 0,
+ DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK = 1,
+};
+
+enum class NumberDataPoint : protozero::pbf_tag_type
+{
+ repeated_KeyValue_attributes = 7,
+ fixed64_start_time_unix_nano = 2,
+ fixed64_time_unix_nano = 3,
+
+ oneof_value_double_as_double = 4,
+ oneof_value_sfixed64_as_int = 6,
+
+ repeated_Exemplar_exemplars = 5,
+ uint32_flags = 8
+};
+
+enum class HistogramDataPoint : protozero::pbf_tag_type
+{
+ repeated_KeyValue_attributes = 9,
+ fixed64_start_time_unix_nano = 2,
+ fixed64_time_unix_nano = 3,
+ fixed64_count = 4,
+ optional_double_sum = 5,
+ repeated_fixed64_bucket_counts = 6,
+ repeated_double_explicit_bounds = 7,
+ repeated_Exemplar_exemplars = 8,
+ uint32_flags = 10,
+ optional_double_min = 11,
+ optional_double_max = 12
+};
+
+enum class Buckets : protozero::pbf_tag_type
+{
+ sint32_offset = 1,
+ repeated_uint64_bucket_counts = 2
+};
+
+enum class ExponentialHistogramDataPoint : protozero::pbf_tag_type
+{
+ repeated_KeyValue_attributes = 1,
+ fixed64_start_time_unix_nano = 2,
+ fixed64_time_unix_nano = 3,
+ fixed64_count = 4,
+ optional_double_sum = 5,
+ sint32_scale = 6,
+ fixed64_zero_count = 7,
+ Buckets_positive = 8,
+ Buckets_negative = 9,
+ uint32_flags = 10,
+ repeated_Exemplar_exemplars = 11,
+ optional_double_min = 12,
+ optional_double_max = 13,
+ double_zero_threshold = 14
+};
+
+enum class ValueAtQuantile : protozero::pbf_tag_type
+{
+ double_quantile = 1,
+ double_value = 2
+};
+
+enum class SummaryDataPoint : protozero::pbf_tag_type
+{
+ repeated_KeyValue_attributes = 7,
+ fixed64_start_time_unix_nano = 2,
+ fixed64_time_unix_nano = 3,
+ fixed64_count = 4,
+ double_sum = 5,
+ repeated_ValueAtQuantile_quantile_values = 6,
+ uint32_flags = 8
+};
+
+enum class Exemplar : protozero::pbf_tag_type
+{
+ repeated_KeyValue_filtered_attributes = 7,
+ fixed64_time_unix_nano = 2,
+ oneof_value_double_as_double = 3,
+ oneof_value_sfixed64_as_int = 6,
+ bytes_span_id = 4,
+ bytes_trace_id = 5
+};
+
+} // namespace zen::otel \ No newline at end of file
diff --git a/src/zentelemetry/otelprotozero.h b/src/zentelemetry/otelprotozero.h
new file mode 100644
index 000000000..8ad69e766
--- /dev/null
+++ b/src/zentelemetry/otelprotozero.h
@@ -0,0 +1,166 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <protozero/types.hpp>
+
+//////////////////////////////////////////////////////////////////////////
+//
+// OTEL .proto definitions, for reference
+//
+
+#if 0
+
+// clang-format off
+
+////////////////////////////////////////////////////////////////////////
+// resource/v1/resource.proto
+//
+
+// Resource information.
+message Resource {
+ // Set of attributes that describe the resource.
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ //
+ // The attribute values SHOULD NOT contain empty values.
+ // The attribute values SHOULD NOT contain bytes values.
+ // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+ // double values.
+ // The attribute values SHOULD NOT contain kvlist values.
+ // The behavior of software that receives attributes containing such values can be unpredictable.
+ // These restrictions can change in a minor release.
+ // The restrictions take origin from the OpenTelemetry specification:
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+ repeated opentelemetry.proto.common.v1.KeyValue attributes = 1;
+
+ // The number of dropped attributes. If the value is 0, then
+ // no attributes were dropped.
+ uint32 dropped_attributes_count = 2;
+
+ // Set of entities that participate in this Resource.
+ //
+ // Note: keys in the references MUST exist in attributes of this message.
+ //
+ // Status: [Development]
+ repeated opentelemetry.proto.common.v1.EntityRef entity_refs = 3;
+}
+
+/////////////////////////////////////////////////////////////////////////
+// common/v1/commmon.proto
+//
+
+// Represents any type of attribute value. AnyValue may contain a
+// primitive value such as a string or integer or it may contain an arbitrary nested
+// object containing arrays, key-value lists and primitives.
+message AnyValue {
+ // The value is one of the listed fields. It is valid for all values to be unspecified
+ // in which case this AnyValue is considered to be "empty".
+ oneof value {
+ string string_value = 1;
+ bool bool_value = 2;
+ int64 int_value = 3;
+ double double_value = 4;
+ ArrayValue array_value = 5;
+ KeyValueList kvlist_value = 6;
+ bytes bytes_value = 7;
+ }
+}
+
+// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message
+// since oneof in AnyValue does not allow repeated fields.
+message ArrayValue {
+ // Array of values. The array may be empty (contain 0 elements).
+ repeated AnyValue values = 1;
+}
+
+// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message
+// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need
+// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to
+// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches
+// are semantically equivalent.
+message KeyValueList {
+ // A collection of key/value pairs of key-value pairs. The list may be empty (may
+ // contain 0 elements).
+ // The keys MUST be unique (it is not allowed to have more than one
+ // value with the same key).
+ repeated KeyValue values = 1;
+}
+
+// Represents a key-value pair that is used to store Span attributes, Link
+// attributes, etc.
+message KeyValue {
+ // The key name of the pair.
+ string key = 1;
+
+ // The value of the pair.
+ AnyValue value = 2;
+}
+
+// InstrumentationScope is a message representing the instrumentation scope information
+// such as the fully qualified name and version.
+message InstrumentationScope {
+ // A name denoting the Instrumentation scope.
+ // An empty instrumentation scope name means the name is unknown.
+ string name = 1;
+
+ // Defines the version of the instrumentation scope.
+ // An empty instrumentation scope version means the version is unknown.
+ string version = 2;
+
+ // Additional attributes that describe the scope. [Optional].
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ repeated KeyValue attributes = 3;
+
+ // The number of attributes that were discarded. Attributes
+ // can be discarded because their keys are too long or because there are too many
+ // attributes. If this value is 0, then no attributes were dropped.
+ uint32 dropped_attributes_count = 4;
+}
+
+// clang-format on
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace zen::otel {
+
+enum class KeyValueList : protozero::pbf_tag_type
+{
+ repeated_KeyValue_values = 1
+};
+
+enum class KeyValue : protozero::pbf_tag_type
+{
+ string_key = 1,
+ AnyValue_value = 2
+};
+
+enum class Resource : protozero::pbf_tag_type
+{
+ repeated_KeyValue_attributes = 1,
+ uint32_dropped_attributes_count = 2,
+ repeated_EntityRef_entity_refs = 3
+};
+
+enum class AnyValue : protozero::pbf_tag_type
+{
+ string_string_value = 1,
+ bool_bool_value = 2,
+ int64_int_value = 3,
+ double_double_value = 4,
+ ArrayValue_array_value = 5,
+ KeyValueList_kvlist_value = 6,
+ bytes_bytes_value = 7
+};
+
+enum class InstrumentationScope : protozero::pbf_tag_type
+{
+ string_name = 1,
+ string_version = 2,
+ repeated_KeyValue_attributes = 3,
+ uint32_dropped_attributes_count = 4
+};
+
+} // namespace zen::otel
diff --git a/src/zentelemetry/oteltraceprotozero.h b/src/zentelemetry/oteltraceprotozero.h
new file mode 100644
index 000000000..8b80c8f7f
--- /dev/null
+++ b/src/zentelemetry/oteltraceprotozero.h
@@ -0,0 +1,474 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "otelprotozero.h"
+
+//////////////////////////////////////////////////////////////////////////
+//
+// OTEL .proto definitions, for reference
+//
+
+#if 0
+
+// clang-format off
+
+/////////////////////////////////////////////////////////////////////////
+// trace/v1/trace.proto
+//
+
+// TracesData represents the traces data that can be stored in a persistent storage,
+// OR can be embedded by other protocols that transfer OTLP traces data but do
+// not implement the OTLP protocol.
+//
+// The main difference between this message and collector protocol is that
+// in this message there will not be any "control" or "metadata" specific to
+// OTLP protocol.
+//
+// When new fields are added into this message, the OTLP request MUST be updated
+// as well.
+message TracesData {
+ // An array of ResourceSpans.
+ // For data coming from a single resource this array will typically contain
+ // one element. Intermediary nodes that receive data from multiple origins
+ // typically batch the data before forwarding further and in that case this
+ // array will contain multiple elements.
+ repeated ResourceSpans resource_spans = 1;
+}
+
+// A collection of ScopeSpans from a Resource.
+message ResourceSpans {
+ reserved 1000;
+
+ // The resource for the spans in this message.
+ // If this field is not set then no resource info is known.
+ opentelemetry.proto.resource.v1.Resource resource = 1;
+
+ // A list of ScopeSpans that originate from a resource.
+ repeated ScopeSpans scope_spans = 2;
+
+ // The Schema URL, if known. This is the identifier of the Schema that the resource data
+ // is recorded in. Notably, the last part of the URL path is the version number of the
+ // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see
+ // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url
+ // This schema_url applies to the data in the "resource" field. It does not apply
+ // to the data in the "scope_spans" field which have their own schema_url field.
+ string schema_url = 3;
+}
+
+// A collection of Spans produced by an InstrumentationScope.
+message ScopeSpans {
+ // The instrumentation scope information for the spans in this message.
+ // Semantically when InstrumentationScope isn't set, it is equivalent with
+ // an empty instrumentation scope name (unknown).
+ opentelemetry.proto.common.v1.InstrumentationScope scope = 1;
+
+ // A list of Spans that originate from an instrumentation scope.
+ repeated Span spans = 2;
+
+ // The Schema URL, if known. This is the identifier of the Schema that the span data
+ // is recorded in. Notably, the last part of the URL path is the version number of the
+ // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see
+ // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url
+ // This schema_url applies to all spans and span events in the "spans" field.
+ string schema_url = 3;
+}
+
+// A Span represents a single operation performed by a single component of the system.
+//
+// The next available field id is 17.
+message Span {
+ // A unique identifier for a trace. All spans from the same trace share
+ // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR
+ // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON
+ // is zero-length and thus is also invalid).
+ //
+ // This field is required.
+ bytes trace_id = 1;
+
+ // A unique identifier for a span within a trace, assigned when the span
+ // is created. The ID is an 8-byte array. An ID with all zeroes OR of length
+ // other than 8 bytes is considered invalid (empty string in OTLP/JSON
+ // is zero-length and thus is also invalid).
+ //
+ // This field is required.
+ bytes span_id = 2;
+
+ // trace_state conveys information about request position in multiple distributed tracing graphs.
+ // It is a trace_state in w3c-trace-context format: https://www.w3.org/TR/trace-context/#tracestate-header
+ // See also https://github.com/w3c/distributed-tracing for more details about this field.
+ string trace_state = 3;
+
+ // The `span_id` of this span's parent span. If this is a root span, then this
+ // field must be empty. The ID is an 8-byte array.
+ bytes parent_span_id = 4;
+
+ // Flags, a bit field.
+ //
+ // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace
+ // Context specification. To read the 8-bit W3C trace flag, use
+ // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`.
+ //
+ // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions.
+ //
+ // Bits 8 and 9 represent the 3 states of whether a span's parent
+ // is remote. The states are (unknown, is not remote, is remote).
+ // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`.
+ // To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`.
+ //
+ // When creating span messages, if the message is logically forwarded from another source
+ // with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD
+ // be copied as-is. If creating from a source that does not have an equivalent flags field
+ // (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST
+ // be set to zero.
+ // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero.
+ //
+ // [Optional].
+ fixed32 flags = 16;
+
+ // A description of the span's operation.
+ //
+ // For example, the name can be a qualified method name or a file name
+ // and a line number where the operation is called. A best practice is to use
+ // the same display name at the same call point in an application.
+ // This makes it easier to correlate spans in different traces.
+ //
+ // This field is semantically required to be set to non-empty string.
+ // Empty value is equivalent to an unknown span name.
+ //
+ // This field is required.
+ string name = 5;
+
+ // SpanKind is the type of span. Can be used to specify additional relationships between spans
+ // in addition to a parent/child relationship.
+ enum SpanKind {
+ // Unspecified. Do NOT use as default.
+ // Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED.
+ SPAN_KIND_UNSPECIFIED = 0;
+
+ // Indicates that the span represents an internal operation within an application,
+ // as opposed to an operation happening at the boundaries. Default value.
+ SPAN_KIND_INTERNAL = 1;
+
+ // Indicates that the span covers server-side handling of an RPC or other
+ // remote network request.
+ SPAN_KIND_SERVER = 2;
+
+ // Indicates that the span describes a request to some remote service.
+ SPAN_KIND_CLIENT = 3;
+
+ // Indicates that the span describes a producer sending a message to a broker.
+ // Unlike CLIENT and SERVER, there is often no direct critical path latency relationship
+ // between producer and consumer spans. A PRODUCER span ends when the message was accepted
+ // by the broker while the logical processing of the message might span a much longer time.
+ SPAN_KIND_PRODUCER = 4;
+
+ // Indicates that the span describes consumer receiving a message from a broker.
+ // Like the PRODUCER kind, there is often no direct critical path latency relationship
+ // between producer and consumer spans.
+ SPAN_KIND_CONSUMER = 5;
+ }
+
+ // Distinguishes between spans generated in a particular context. For example,
+ // two spans with the same name may be distinguished using `CLIENT` (caller)
+ // and `SERVER` (callee) to identify queueing latency associated with the span.
+ SpanKind kind = 6;
+
+ // The start time of the span. On the client side, this is the time
+ // kept by the local machine where the span execution starts. On the server side, this
+ // is the time when the server's application handler starts running.
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970.
+ //
+ // This field is semantically required and it is expected that end_time >= start_time.
+ fixed64 start_time_unix_nano = 7;
+
+ // The end time of the span. On the client side, this is the time
+ // kept by the local machine where the span execution ends. On the server side, this
+ // is the time when the server application handler stops running.
+ // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970.
+ //
+ // This field is semantically required and it is expected that end_time >= start_time.
+ fixed64 end_time_unix_nano = 8;
+
+ // A collection of key/value pairs. Note, global attributes
+ // like server name can be set using the resource API. Examples of attributes:
+ //
+ // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
+ // "/http/server_latency": 300
+ // "example.com/myattribute": true
+ // "example.com/score": 10.239
+ //
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ //
+ // The attribute values SHOULD NOT contain empty values.
+ // The attribute values SHOULD NOT contain bytes values.
+ // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+ // double values.
+ // The attribute values SHOULD NOT contain kvlist values.
+ // The behavior of software that receives attributes containing such values can be unpredictable.
+ // These restrictions can change in a minor release.
+ // The restrictions take origin from the OpenTelemetry specification:
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+ repeated opentelemetry.proto.common.v1.KeyValue attributes = 9;
+
+ // The number of attributes that were discarded. Attributes
+ // can be discarded because their keys are too long or because there are too many
+ // attributes. If this value is 0, then no attributes were dropped.
+ uint32 dropped_attributes_count = 10;
+
+ // Event is a time-stamped annotation of the span, consisting of user-supplied
+ // text description and key-value pairs.
+ message Event {
+ // The time the event occurred.
+ fixed64 time_unix_nano = 1;
+
+ // The name of the event.
+ // This field is semantically required to be set to non-empty string.
+ string name = 2;
+
+ // A collection of attribute key/value pairs on the event.
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ //
+ // The attribute values SHOULD NOT contain empty values.
+ // The attribute values SHOULD NOT contain bytes values.
+ // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+ // double values.
+ // The attribute values SHOULD NOT contain kvlist values.
+ // The behavior of software that receives attributes containing such values can be unpredictable.
+ // These restrictions can change in a minor release.
+ // The restrictions take origin from the OpenTelemetry specification:
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+ repeated opentelemetry.proto.common.v1.KeyValue attributes = 3;
+
+ // The number of dropped attributes. If the value is 0,
+ // then no attributes were dropped.
+ uint32 dropped_attributes_count = 4;
+ }
+
+ // A collection of Event items.
+ repeated Event events = 11;
+
+ // The number of dropped events. If the value is 0, then no
+ // events were dropped.
+ uint32 dropped_events_count = 12;
+
+ // A pointer from the current span to another span in the same trace or in a
+ // different trace. For example, this can be used in batching operations,
+ // where a single batch handler processes multiple requests from different
+ // traces or when the handler receives a request from a different project.
+ message Link {
+ // A unique identifier of a trace that this linked span is part of. The ID is a
+ // 16-byte array.
+ bytes trace_id = 1;
+
+ // A unique identifier for the linked span. The ID is an 8-byte array.
+ bytes span_id = 2;
+
+ // The trace_state associated with the link.
+ string trace_state = 3;
+
+ // A collection of attribute key/value pairs on the link.
+ // Attribute keys MUST be unique (it is not allowed to have more than one
+ // attribute with the same key).
+ //
+ // The attribute values SHOULD NOT contain empty values.
+ // The attribute values SHOULD NOT contain bytes values.
+ // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+ // double values.
+ // The attribute values SHOULD NOT contain kvlist values.
+ // The behavior of software that receives attributes containing such values can be unpredictable.
+ // These restrictions can change in a minor release.
+ // The restrictions take origin from the OpenTelemetry specification:
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+ repeated opentelemetry.proto.common.v1.KeyValue attributes = 4;
+
+ // The number of dropped attributes. If the value is 0,
+ // then no attributes were dropped.
+ uint32 dropped_attributes_count = 5;
+
+ // Flags, a bit field.
+ //
+ // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace
+ // Context specification. To read the 8-bit W3C trace flag, use
+ // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`.
+ //
+ // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions.
+ //
+ // Bits 8 and 9 represent the 3 states of whether the link is remote.
+ // The states are (unknown, is not remote, is remote).
+ // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`.
+ // To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`.
+ //
+ // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero.
+ // When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero.
+ //
+ // [Optional].
+ fixed32 flags = 6;
+ }
+
+ // A collection of Links, which are references from this span to a span
+ // in the same or different trace.
+ repeated Link links = 13;
+
+ // The number of dropped links after the maximum size was
+ // enforced. If this value is 0, then no links were dropped.
+ uint32 dropped_links_count = 14;
+
+ // An optional final status for this span. Semantically when Status isn't set, it means
+ // span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0).
+ Status status = 15;
+}
+
+// The Status type defines a logical error model that is suitable for different
+// programming environments, including REST APIs and RPC APIs.
+message Status {
+ reserved 1;
+
+ // A developer-facing human readable error message.
+ string message = 2;
+
+ // For the semantics of status codes see
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status
+ enum StatusCode {
+ // The default status.
+ STATUS_CODE_UNSET = 0;
+ // The Span has been validated by an Application developer or Operator to
+ // have completed successfully.
+ STATUS_CODE_OK = 1;
+ // The Span contains an error.
+ STATUS_CODE_ERROR = 2;
+ };
+
+ // The status code.
+ StatusCode code = 3;
+}
+
+// SpanFlags represents constants used to interpret the
+// Span.flags field, which is protobuf 'fixed32' type and is to
+// be used as bit-fields. Each non-zero value defined in this enum is
+// a bit-mask. To extract the bit-field, for example, use an
+// expression like:
+//
+// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK)
+//
+// See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions.
+//
+// Note that Span flags were introduced in version 1.1 of the
+// OpenTelemetry protocol. Older Span producers do not set this
+// field, consequently consumers should not rely on the absence of a
+// particular flag bit to indicate the presence of a particular feature.
+enum SpanFlags {
+ // The zero value for the enum. Should not be used for comparisons.
+ // Instead use bitwise "and" with the appropriate mask as shown above.
+ SPAN_FLAGS_DO_NOT_USE = 0;
+
+ // Bits 0-7 are used for trace flags.
+ SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF;
+
+ // Bits 8 and 9 are used to indicate that the parent span or link span is remote.
+ // Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known.
+ // Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote.
+ SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100;
+ SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200;
+
+ // Bits 10-31 are reserved for future use.
+}
+
+// clang-format on
+#endif
+
+namespace zen::otel::pbf {
+
+// Traces
+
+enum class TracesData : protozero::pbf_tag_type
+{
+ repeated_ResourceSpans_resource_spans = 1
+};
+
+enum class ResourceSpans : protozero::pbf_tag_type
+{
+ Resource_resource = 1,
+ repeated_ScopeSpans_scope_spans = 2,
+ string_schema_url = 3
+};
+
+enum class ScopeSpans : protozero::pbf_tag_type
+{
+ InstrumentationScope_scope = 1,
+ repeated_Span_spans = 2,
+ string_schema_url = 3
+};
+
+enum SpanFlags
+{
+ SPAN_FLAGS_DO_NOT_USE = 0,
+ SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF,
+ SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100,
+ SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200,
+};
+
+enum SpanKind
+{
+ SPAN_KIND_UNSPECIFIED = 0,
+ SPAN_KIND_INTERNAL = 1,
+ SPAN_KIND_SERVER = 2,
+ SPAN_KIND_CLIENT = 3,
+ SPAN_KIND_PRODUCER = 4,
+ SPAN_KIND_CONSUMER = 5
+};
+
+enum class Span_Event : protozero::pbf_tag_type
+{
+ fixed64_time_unix_nano = 1,
+ string_name = 2,
+ repeated_KeyValue_attributes = 3,
+ uint32_dropped_attributes_count = 4
+};
+
+enum class Span_Link : protozero::pbf_tag_type
+{
+ bytes_trace_id = 1,
+ bytes_span_id = 2,
+ string_trace_state = 3,
+ repeated_KeyValue_attributes = 4,
+ uint32_dropped_attributes_count = 5,
+ fixed32_flags = 6
+};
+
+enum class Span : protozero::pbf_tag_type
+{
+ required_bytes_trace_id = 1,
+ required_bytes_span_id = 2,
+ string_trace_state = 3,
+ bytes_parent_span_id = 4,
+ fixed32_flags = 16,
+ required_string_name = 5,
+ SpanKind_kind = 6,
+ required_fixed64_start_time_unix_nano = 7,
+ required_fixed64_end_time_unix_nano = 8,
+ repeated_KeyValue_attributes = 9,
+ uint32_dropped_attributes_count = 10,
+ repeated_Event_events = 11,
+ uint32_dropped_events_count = 12,
+ repeated_Link_links = 13,
+ uint32_dropped_links_count = 14,
+ Status_status = 15
+};
+
+enum StatusCode
+{
+ STATUS_CODE_UNSET = 0,
+ STATUS_CODE_OK = 1,
+ STATUS_CODE_ERROR = 2
+};
+
+enum class Status : protozero::pbf_tag_type
+{
+ string_message = 2,
+ StatusCode_code = 3
+};
+
+} // namespace zen::otel::pbf
diff --git a/src/zentelemetry/otlpencoder.cpp b/src/zentelemetry/otlpencoder.cpp
new file mode 100644
index 000000000..677545066
--- /dev/null
+++ b/src/zentelemetry/otlpencoder.cpp
@@ -0,0 +1,478 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zentelemetry/otlpencoder.h"
+
+#include <zenbase/zenbase.h>
+#include <zentelemetry/otlptrace.h>
+
+#include <spdlog/sinks/sink.h>
+#include <zencore/testing.h>
+
+#include <protozero/buffer_string.hpp>
+#include <protozero/pbf_builder.hpp>
+
+#include "otellogprotozero.h"
+#include "otelmetricsprotozero.h"
+#include "otelprotozero.h"
+#include "oteltraceprotozero.h"
+
+#if ZEN_WITH_OTEL
+
+namespace zen {
+
+OtlpEncoder::OtlpEncoder()
+{
+}
+
+OtlpEncoder::~OtlpEncoder()
+{
+}
+
+static int
+MapSeverity(const spdlog::level::level_enum Level)
+{
+ switch (Level)
+ {
+ case spdlog::level::critical:
+ return otel::SEVERITY_NUMBER_FATAL;
+ case spdlog::level::err:
+ return otel::SEVERITY_NUMBER_ERROR;
+ case spdlog::level::warn:
+ return otel::SEVERITY_NUMBER_WARN;
+ case spdlog::level::info:
+ return otel::SEVERITY_NUMBER_INFO;
+ case spdlog::level::debug:
+ return otel::SEVERITY_NUMBER_DEBUG;
+ default:
+ case spdlog::level::trace:
+ return otel::SEVERITY_NUMBER_TRACE;
+ }
+}
+
+static const char*
+MapSeverityText(const spdlog::level::level_enum Level)
+{
+ switch (Level)
+ {
+ case spdlog::level::critical:
+ return "fatal";
+ case spdlog::level::err:
+ return "error";
+ case spdlog::level::warn:
+ return "warn";
+ case spdlog::level::info:
+ return "info";
+ case spdlog::level::debug:
+ return "debug";
+ default:
+ case spdlog::level::trace:
+ return "trace";
+ }
+}
+
+std::string
+OtlpEncoder::FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const
+{
+ std::string Data;
+
+ // LogsData
+ {
+ protozero::pbf_builder<otel::LogsData> Builder{Data};
+
+ // ResourceLogs
+ {
+ protozero::pbf_builder<otel::ResourceLogs> RlBuilder{Builder, otel::LogsData::required_repeated_ResourceLogs_resource_logs};
+
+ // ResourceLogs / Resource
+ {
+ protozero::pbf_builder<otel::Resource> Res{RlBuilder, otel::ResourceLogs::optional_Resource_resource};
+
+ AppendResourceAttributes(Res);
+ }
+
+ // ScopeLogs scope_logs
+ {
+ protozero::pbf_builder<otel::ScopeLogs> SlBuilder{RlBuilder, otel::ResourceLogs::required_repeated_ScopeLogs_scope_logs};
+
+ {
+ protozero::pbf_builder<otel::InstrumentationScope> IsBuilder{SlBuilder,
+ otel::ScopeLogs::required_InstrumentationScope_scope};
+
+ IsBuilder.add_string(otel::InstrumentationScope::string_name, Msg.logger_name.data(), Msg.logger_name.size());
+ }
+
+ // LogRecord log_records
+ {
+ protozero::pbf_builder<otel::LogRecord> LrBuilder{SlBuilder, otel::ScopeLogs::required_repeated_LogRecord_log_records};
+
+ LrBuilder.add_fixed64(otel::LogRecord::required_fixed64_time_unix_nano,
+ std::chrono::duration_cast<std::chrono::nanoseconds>(Msg.time.time_since_epoch()).count());
+
+ const int Severity = MapSeverity(Msg.level);
+
+ LrBuilder.add_enum(otel::LogRecord::optional_SeverityNumber_severity_number, Severity);
+
+ LrBuilder.add_string(otel::LogRecord::optional_string_severity_text, MapSeverityText(Msg.level));
+
+ otel::TraceId TraceId;
+ const otel::SpanId SpanId = otel::Span::GetCurrentSpanId(TraceId);
+
+ if (SpanId && TraceId)
+ {
+ LrBuilder.add_bytes(otel::LogRecord::optional_bytes_trace_id, TraceId.GetData(), TraceId.kSize);
+ LrBuilder.add_bytes(otel::LogRecord::optional_bytes_span_id, SpanId.GetData(), SpanId.kSize);
+ }
+
+ // body
+ {
+ protozero::pbf_builder<otel::AnyValue> BodyBuilder{LrBuilder, otel::LogRecord::optional_anyvalue_body};
+
+ BodyBuilder.add_string(otel::AnyValue::string_string_value, Msg.payload.data(), Msg.payload.size());
+ }
+
+ // attributes
+
+ {
+ protozero::pbf_builder<otel::KeyValue> KvBuilder{LrBuilder, otel::LogRecord::optional_repeated_kv_attributes};
+ KvBuilder.add_string(otel::KeyValue::string_key, "thread_id");
+
+ {
+ protozero::pbf_builder<otel::AnyValue> AvBuilder{KvBuilder, otel::KeyValue::AnyValue_value};
+
+ AvBuilder.add_int64(otel::AnyValue::int64_int_value, Msg.thread_id);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ return Data;
+}
+
+std::string
+OtlpEncoder::FormatOtelMetrics() const
+{
+ std::string Data;
+
+# if 0
+ static int64_t LastNanos = 0;
+
+ const int64_t NowNanos =
+ std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+
+ // MetricsData
+ {
+ protozero::pbf_builder<otel::MetricsData> Builder{Data};
+
+ // ResourceMetrics
+ protozero::pbf_builder<otel::ResourceMetrics> Rm{Builder, otel::MetricsData::repeated_ResourceMetrics_resource_metrics};
+
+ {
+ protozero::pbf_builder<otel::Resource> Res{Rm, otel::ResourceMetrics::Resource_resource};
+
+ AppendResourceAttributes(Res);
+ }
+
+ // ScopeMetrics
+ protozero::pbf_builder<otel::ScopeMetrics> Sm{Rm, otel::ResourceMetrics::repeated_ScopeMetrics_scope_metrics};
+
+ {
+ // InstrumentationScope
+ protozero::pbf_builder<otel::InstrumentationScope> Is{Sm, otel::ScopeMetrics::InstrumentationScope_scope};
+ Is.add_string(otel::InstrumentationScope::string_name, "scope_name");
+ }
+
+ {
+ protozero::pbf_builder<otel::Metric> Metric{Sm, otel::ScopeMetrics::repeated_Metric_metrics};
+ Metric.add_string(otel::Metric::string_name, "metric_name");
+ Metric.add_string(otel::Metric::string_unit, "KiB");
+
+ // Gauge
+ {
+ protozero::pbf_builder<otel::Gauge> Gauge{Metric, otel::Metric::oneof_data_Gauge_gauge};
+
+ protozero::pbf_builder<otel::NumberDataPoint> Dp{Gauge, otel::Gauge::repeated_NumberDataPoint_data_points};
+ Dp.add_fixed64(otel::NumberDataPoint::fixed64_time_unix_nano, NowNanos);
+ Dp.add_fixed64(otel::NumberDataPoint::fixed64_start_time_unix_nano, LastNanos);
+ Dp.add_sfixed64(otel::NumberDataPoint::oneof_value_sfixed64_as_int, rand() * 470 / RAND_MAX);
+ }
+ }
+
+ const int RequestCount = rand() % 600;
+
+ {
+ protozero::pbf_builder<otel::Metric> Metric{Sm, otel::ScopeMetrics::repeated_Metric_metrics};
+ Metric.add_string(otel::Metric::string_name, "request_count");
+ Metric.add_string(otel::Metric::string_unit, "requests");
+
+ static int SumValue = 0;
+ SumValue += RequestCount;
+
+ // Sum
+ {
+ protozero::pbf_builder<otel::Sum> Sum{Metric, otel::Metric::oneof_data_Sum_sum};
+ Sum.add_enum(otel::Sum::AggregationTemporality_aggregation_temporality, otel::AGGREGATION_TEMPORALITY_CUMULATIVE);
+ Sum.add_bool(otel::Sum::bool_is_monotonic, true);
+
+ protozero::pbf_builder<otel::NumberDataPoint> Dp{Sum, otel::Sum::repeated_NumberDataPoint_data_points};
+ Dp.add_fixed64(otel::NumberDataPoint::fixed64_time_unix_nano, NowNanos);
+ Dp.add_fixed64(otel::NumberDataPoint::fixed64_start_time_unix_nano, LastNanos);
+ Dp.add_double(otel::NumberDataPoint::oneof_value_double_as_double, SumValue);
+ }
+ }
+
+ {
+ protozero::pbf_builder<otel::Metric> Metric{Sm, otel::ScopeMetrics::repeated_Metric_metrics};
+ Metric.add_string(otel::Metric::string_name, "request_latency");
+ Metric.add_string(otel::Metric::string_unit, "ms");
+
+ // Histogram
+ {
+ protozero::pbf_builder<otel::Histogram> Histogram{Metric, otel::Metric::oneof_data_Histogram_histogram};
+ Histogram.add_enum(otel::Histogram::AggregationTemporality_aggregation_temporality,
+ otel::AGGREGATION_TEMPORALITY_CUMULATIVE);
+
+ protozero::pbf_builder<otel::HistogramDataPoint> Dp{Histogram, otel::Histogram::repeated_HistogramDataPoint_data_points};
+ Dp.add_fixed64(otel::HistogramDataPoint::fixed64_time_unix_nano, NowNanos);
+ Dp.add_fixed64(otel::HistogramDataPoint::fixed64_start_time_unix_nano, LastNanos);
+
+ // Simulated latency value
+
+ double Sum = 0, Min = 0, Max = 0;
+ int Buckets[6] = {0};
+
+ for (int i = 0; i < RequestCount; ++i)
+ {
+ const int Latency = rand() % 250;
+
+ if (i == 0 || Latency < Min)
+ {
+ Min = Latency;
+ }
+ if (i == 0 || Latency > Max)
+ {
+ Max = Latency;
+ }
+
+ Sum += Latency;
+
+ if (Latency >= 0 && Latency < 10)
+ {
+ ++Buckets[0]; // [0,10)
+ }
+ else if (Latency >= 10 && Latency < 25)
+ {
+ ++Buckets[1]; // [10,25)
+ }
+ else if (Latency >= 25 && Latency < 50)
+ {
+ ++Buckets[2]; // [25,50)
+ }
+ else if (Latency >= 50 && Latency < 100)
+ {
+ ++Buckets[3]; // [50,100)
+ }
+ else if (Latency >= 100 && Latency < 250)
+ {
+ ++Buckets[4]; // [100,250)
+ }
+ else
+ {
+ ++Buckets[5]; // [250,+inf)
+ }
+ }
+
+ Dp.add_fixed64(otel::HistogramDataPoint::fixed64_count, RequestCount);
+ Dp.add_double(otel::HistogramDataPoint::optional_double_sum, Sum);
+ Dp.add_double(otel::HistogramDataPoint::optional_double_min, Min);
+ Dp.add_double(otel::HistogramDataPoint::optional_double_max, Max);
+
+ // Bucket bounds
+ Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 10.0);
+ Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 25.0);
+ Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 50.0);
+ Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 100.0);
+ Dp.add_double(otel::HistogramDataPoint::repeated_double_explicit_bounds, 250.0);
+
+ // Buckets
+ for (int i = 0; i < 6; ++i)
+ {
+ Dp.add_fixed64(otel::HistogramDataPoint::repeated_fixed64_bucket_counts, Buckets[i]);
+ }
+ }
+ }
+
+# if 0
+ {
+ protozero::pbf_builder<otel::Metric> Metric{Sm, otel::ScopeMetrics::repeated_Metric_metrics};
+ Metric.add_string(otel::Metric::string_name, "request_payload");
+ Metric.add_string(otel::Metric::string_unit, "KiB");
+
+ // ExponentialHistogram
+ {
+ protozero::pbf_builder<otel::ExponentialHistogram> ExponentialHistogram{
+ Metric, otel::Metric::oneof_data_ExponentialHistogram_exponential_histogram};
+ ExponentialHistogram.add_enum(otel::ExponentialHistogram::AggregationTemporality_aggregation_temporality,
+ otel::AGGREGATION_TEMPORALITY_CUMULATIVE);
+
+ protozero::pbf_builder<otel::ExponentialHistogramDataPoint> Dp{
+ ExponentialHistogram, otel::ExponentialHistogram::repeated_ExponentialHistogramDataPoint_data_points};
+ Dp.add_fixed64(otel::ExponentialHistogramDataPoint::fixed64_time_unix_nano, NowNanos);
+ Dp.add_fixed64(otel::ExponentialHistogramDataPoint::fixed64_start_time_unix_nano, LastNanos);
+
+ // Simulated payload size values
+
+ int64_t sum = 0;
+ int count = 0;
+
+ for (int i = 0; i < RequestCount; ++i)
+ {
+ const int PayloadSize = rand() % 1'500'000; // [0,1500000) bytes
+
+ sum += PayloadSize;
+ ++count;
+ }
+
+ Dp.add_fixed64(otel::ExponentialHistogramDataPoint::fixed64_count, count);
+ Dp.add_double(otel::ExponentialHistogramDataPoint::optional_double_sum, sum);
+ Dp.add_sint32(otel::ExponentialHistogramDataPoint::sint32_scale, 4);
+ }
+ }
+# endif
+ }
+ LastNanos = NowNanos;
+# endif
+ return Data;
+}
+
+void
+OtlpEncoder::AddResourceAttribute(const std::string_view& Key, const std::string_view& Value)
+{
+ m_ResourceLock.WithExclusiveLock([&] { m_ResourceAttributes[std::string(Key)] = std::string(Value); });
+}
+
+void
+OtlpEncoder::AddResourceAttribute(const std::string_view& Key, int64_t Value)
+{
+ m_ResourceLock.WithExclusiveLock([&] { m_ResourceIntAttributes[std::string(Key)] = Value; });
+}
+
+void
+OtlpEncoder::AppendResourceAttributes(protozero::pbf_builder<otel::Resource>& Res) const
+{
+ using namespace otel;
+
+ m_ResourceLock.WithSharedLock([&] {
+ for (auto const& [K, V] : m_ResourceAttributes)
+ {
+ protozero::pbf_builder<otel::KeyValue> KvBuilder{Res, otel::Resource::repeated_KeyValue_attributes};
+ KvBuilder.add_string(otel::KeyValue::string_key, K.c_str());
+ protozero::pbf_builder<otel::AnyValue> AnyBuilder{KvBuilder, otel::KeyValue::AnyValue_value};
+ AnyBuilder.add_string(otel::AnyValue::string_string_value, V.c_str());
+ }
+
+ for (auto const& [K, V] : m_ResourceIntAttributes)
+ {
+ protozero::pbf_builder<otel::KeyValue> KvBuilder{Res, otel::Resource::repeated_KeyValue_attributes};
+ KvBuilder.add_string(otel::KeyValue::string_key, K.c_str());
+ protozero::pbf_builder<otel::AnyValue> AnyBuilder{KvBuilder, otel::KeyValue::AnyValue_value};
+ AnyBuilder.add_int64(otel::AnyValue::int64_int_value, V);
+ }
+ });
+}
+
+template<typename ParentBuilder>
+static void
+AppendAttributesToBuilder(ParentBuilder& Parent, auto KeyValueField, const otel::AttributePair* Attrs)
+{
+ for (const otel::AttributePair* Attr = Attrs; Attr != nullptr; Attr = Attr->Next)
+ {
+ protozero::pbf_builder<otel::KeyValue> KvBuilder{Parent, KeyValueField};
+ KvBuilder.add_string(otel::KeyValue::string_key, Attr->Key);
+
+ protozero::pbf_builder<otel::AnyValue> AnyBuilder{KvBuilder, otel::KeyValue::AnyValue_value};
+
+ if (Attr->IsNumeric())
+ {
+ AnyBuilder.add_int64(otel::AnyValue::int64_int_value, Attr->GetNumericValue());
+ }
+ else if (Attr->IsString())
+ {
+ AnyBuilder.add_string(otel::AnyValue::string_string_value, Attr->GetStringValue());
+ }
+ }
+}
+
+std::string
+OtlpEncoder::FormatOtelTrace(zen::otel::TraceId Id, std::span<const zen::otel::Span*> Spans) const
+{
+ std::string Data;
+
+ using namespace otel;
+
+ // TracesData
+ {
+ protozero::pbf_builder<pbf::TracesData> Builder{Data};
+
+ {
+ protozero::pbf_builder<pbf::ResourceSpans> Rs{Builder, pbf::TracesData::repeated_ResourceSpans_resource_spans};
+
+ {
+ protozero::pbf_builder<otel::Resource> Res{Rs, pbf::ResourceSpans::Resource_resource};
+
+ AppendResourceAttributes(Res);
+ }
+
+ for (const zen::otel::Span* SpanPtr : Spans)
+ {
+ protozero::pbf_builder<pbf::ScopeSpans> Ss{Rs, pbf::ResourceSpans::repeated_ScopeSpans_scope_spans};
+
+ {
+ // InstrumentationScope
+ protozero::pbf_builder<otel::InstrumentationScope> Is{Ss, pbf::ScopeSpans::InstrumentationScope_scope};
+ Is.add_string(otel::InstrumentationScope::string_name, "scope_name");
+ }
+
+ const SpanId ThisSpanId = SpanPtr->GetSpanId();
+
+ {
+ protozero::pbf_builder<pbf::Span> Sb{Ss, pbf::ScopeSpans::repeated_Span_spans};
+
+ Sb.add_bytes(pbf::Span::required_bytes_trace_id, Id.GetData(), TraceId::kSize);
+ Sb.add_bytes(pbf::Span::required_bytes_span_id, ThisSpanId.GetData(), SpanId::kSize);
+ // Sb.add_string(pbf::Span::string_trace_state, "state-value");
+ //
+ if (const otel::Span* ParentSpan = SpanPtr->GetParentSpan())
+ {
+ const SpanId ParentSpanId = ParentSpan->GetSpanId();
+ Sb.add_bytes(pbf::Span::bytes_parent_span_id, ParentSpanId.GetData(), SpanId::kSize);
+ }
+
+ Sb.add_fixed32(pbf::Span::fixed32_flags, 0);
+ Sb.add_string(pbf::Span::required_string_name, SpanPtr->GetName());
+ Sb.add_enum(pbf::Span::SpanKind_kind, (int)SpanPtr->GetKind());
+ Sb.add_fixed64(pbf::Span::required_fixed64_start_time_unix_nano, SpanPtr->GetStartTime());
+ Sb.add_fixed64(pbf::Span::required_fixed64_end_time_unix_nano, SpanPtr->GetEndTime());
+
+ AppendAttributesToBuilder(Sb, pbf::Span::repeated_KeyValue_attributes, SpanPtr->GetAttributes());
+
+ for (const otel::Event* Event = SpanPtr->GetEvents(); Event != nullptr; Event = Event->NextEvent)
+ {
+ protozero::pbf_builder<pbf::Span_Event> EventBuilder{Sb, pbf::Span::repeated_Event_events};
+ EventBuilder.add_fixed64(pbf::Span_Event::fixed64_time_unix_nano, Event->Timestamp);
+ EventBuilder.add_string(pbf::Span_Event::string_name, Event->Name);
+
+ AppendAttributesToBuilder(EventBuilder, pbf::Span_Event::repeated_KeyValue_attributes, Event->Attributes);
+ }
+ }
+ }
+ }
+ }
+
+ return Data;
+}
+
+} // namespace zen
+
+#endif
diff --git a/src/zentelemetry/otlptrace.cpp b/src/zentelemetry/otlptrace.cpp
new file mode 100644
index 000000000..b634eacf6
--- /dev/null
+++ b/src/zentelemetry/otlptrace.cpp
@@ -0,0 +1,398 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zentelemetry/otlptrace.h"
+#include "oteltraceprotozero.h"
+
+#if ZEN_WITH_OTEL
+
+# include <zencore/endian.h>
+# include <zencore/memory/memoryarena.h>
+# include <zencore/session.h>
+# include <zencore/testing.h>
+# include <zencore/uid.h>
+
+# include <EASTL/list.h>
+# include <EASTL/vector.h>
+
+# include <protozero/pbf_builder.hpp>
+# include <random>
+
+namespace zen::otel {
+
+//////////////////////////////////////////////////////////////////////////
+
+uint64_t
+NowInNanoseconds()
+{
+ const uint64_t NowNanos =
+ std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+ return static_cast<uint64_t>(NowNanos);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+/**
+ * xorshiftr128+ random number generator
+ *
+ * https://en.wikipedia.org/wiki/Xorshift#xorshift+
+ */
+class XorShiftr128p
+{
+public:
+ XorShiftr128p()
+ {
+ std::seed_seq SeedSeq{1, 2, 3, 5};
+ do
+ {
+ SeedSeq.generate(reinterpret_cast<uint32_t*>(m_State), reinterpret_cast<uint32_t*>(&m_State[2]));
+ } while (m_State[0] == 0 && m_State[1] == 0);
+ }
+
+ uint64_t Next()
+ {
+ uint64_t x = m_State[0];
+ uint64_t const y = m_State[1];
+ m_State[0] = y;
+ x ^= x << 23;
+ x ^= x >> 17;
+ x ^= y;
+ m_State[1] = x + y;
+ return x;
+ }
+
+private:
+ uint64_t m_State[2];
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct TraceState
+{
+ Span* ActiveSpan = nullptr;
+ TraceId CurrentTraceId;
+};
+
+thread_local TraceState t_TraceState;
+
+//////////////////////////////////////////////////////////////////////////
+
+Span::Span(MemoryArena& Arena, std::string_view Name, Span* SpanChain)
+: m_Arena(Arena)
+, m_SpanChain(SpanChain)
+, m_SpanId(SpanId::NewSpanId())
+, m_Name(m_Arena.DuplicateString(Name))
+{
+ if (Span* ActiveSpan = t_TraceState.ActiveSpan)
+ {
+ m_ParentSpan = ActiveSpan;
+ m_NextSibling = ActiveSpan->m_FirstChild;
+ ActiveSpan->m_FirstChild = this;
+ }
+ else
+ {
+ m_ParentSpan = nullptr;
+ t_TraceState.CurrentTraceId = TraceId::NewTraceId();
+ }
+
+ t_TraceState.ActiveSpan = this;
+ m_StartTime = NowInNanoseconds();
+}
+
+Span::~Span()
+{
+ End();
+}
+
+void
+Span::End()
+{
+ if (m_Ended)
+ {
+ return;
+ }
+
+ ZEN_ASSERT(t_TraceState.ActiveSpan == this);
+
+ m_Ended = true;
+ t_TraceState.ActiveSpan = m_ParentSpan;
+ m_EndTime = NowInNanoseconds();
+}
+
+Span*
+Span::GetCurrentSpan()
+{
+ return t_TraceState.ActiveSpan;
+}
+
+SpanId
+Span::GetCurrentSpanId(TraceId& OutTraceId)
+{
+ if (t_TraceState.ActiveSpan)
+ {
+ OutTraceId = t_TraceState.CurrentTraceId;
+ return t_TraceState.ActiveSpan->m_SpanId;
+ }
+ else
+ {
+ OutTraceId = TraceId();
+ return SpanId();
+ }
+}
+
+void
+Span::AddEvent(std::string_view Name)
+{
+ Event* NewEvent = new (m_Arena) Event();
+ NewEvent->Name = m_Arena.DuplicateString(Name);
+ NewEvent->Timestamp = NowInNanoseconds();
+ NewEvent->NextEvent = m_Events;
+ m_Events = NewEvent;
+}
+
+void
+Span::AddEvent(std::string_view Name, uint64_t Timestamp)
+{
+ Event* NewEvent = new (m_Arena) Event();
+ NewEvent->Name = m_Arena.DuplicateString(Name);
+ NewEvent->Timestamp = Timestamp;
+ NewEvent->NextEvent = m_Events;
+ m_Events = NewEvent;
+}
+
+void
+Span::AddEvent(std::string_view Name, const AttributeList& Attributes)
+{
+ Event* NewEvent = new (m_Arena) Event();
+ NewEvent->Name = m_Arena.DuplicateString(Name);
+ NewEvent->Timestamp = NowInNanoseconds();
+ NewEvent->NextEvent = m_Events;
+ m_Events = NewEvent;
+
+ for (auto& [K, V] : Attributes)
+ {
+ AttributePair* NewAttr = new (m_Arena) AttributePair();
+ NewAttr->Key = m_Arena.DuplicateString(K);
+ NewAttr->SetStringValue(m_Arena.DuplicateString(V));
+ NewAttr->Next = NewEvent->Attributes;
+ NewEvent->Attributes = NewAttr;
+ }
+}
+
+void
+Span::AddAttribute(std::string_view Key, std::string_view Value)
+{
+ AttributePair* NewAttr = new (m_Arena) AttributePair();
+ NewAttr->Key = m_Arena.DuplicateString(Key);
+ NewAttr->SetStringValue(m_Arena.DuplicateString(Value));
+ NewAttr->Next = m_Attributes;
+ m_Attributes = NewAttr;
+}
+
+void
+Span::AddAttribute(std::string_view Key, uint64_t Value)
+{
+ AttributePair* NewAttr = new (m_Arena) AttributePair();
+ NewAttr->Key = m_Arena.DuplicateString(Key);
+ NewAttr->SetNumericValue(Value);
+ NewAttr->Next = m_Attributes;
+ m_Attributes = NewAttr;
+}
+
+void
+Span::AddAttributes(const AttributeList& Attributes)
+{
+ for (const auto& [K, V] : Attributes)
+ {
+ AddAttribute(K, V);
+ }
+}
+
+void*
+Span::operator new(size_t Size, zen::MemoryArena& Arena)
+{
+ return Arena.Allocate(Size);
+}
+
+void
+Span::operator delete(void* Ptr, zen::MemoryArena& Arena)
+{
+ // This exists because operator new with placement arguments
+ // requires a matching operator delete, but we don't actually
+ // need to do anything here
+ ZEN_UNUSED(Ptr, Arena);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+std::atomic<uint32_t> g_TraceCounter;
+
+TraceId
+TraceId::NewTraceId()
+{
+ uint8_t TraceIdBytes[kSize];
+
+ // We use our session ID as the basis for trace IDs
+ const Oid SessionId = GetSessionId();
+ memcpy(TraceIdBytes, SessionId.OidBits, sizeof SessionId.OidBits);
+
+ const uint32_t TraceCounterBe = ByteSwap(g_TraceCounter.fetch_add(1));
+
+ memcpy(&TraceIdBytes[12], &TraceCounterBe, 4);
+
+ return TraceId(std::span<const uint8_t, kSize>(TraceIdBytes, kSize));
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+SpanId
+SpanId::NewSpanId()
+{
+ // Just use a new OID and take the first 8 bytes. We'll probably want
+ // a more native solution later
+
+ return SpanId(Oid::NewOid());
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+Ref<TraceRecorder> g_TraceRecorder;
+
+void
+SetTraceRecorder(Ref<TraceRecorder> Recorder)
+{
+ g_TraceRecorder = std::move(Recorder);
+}
+
+bool
+IsRecording()
+{
+ return !!g_TraceRecorder;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+thread_local Tracer* t_Tracer;
+
+struct Tracer::Impl
+{
+ Impl() : m_PrevTracer(t_Tracer) {}
+ ~Impl() { t_Tracer = m_PrevTracer; }
+
+ MemoryArena m_Arena;
+ Tracer* m_PrevTracer;
+ Span* m_SpanChain = nullptr;
+ std::atomic<uint64_t> m_SpanCount;
+};
+
+Tracer::Tracer() : m_Impl(new Impl())
+{
+ t_Tracer = this;
+}
+
+Tracer::~Tracer()
+{
+ if (Ref<TraceRecorder> Recorder = g_TraceRecorder)
+ {
+ const uint64_t SpanCount = m_Impl->m_SpanCount.load();
+ const Span** Spans = new (m_Impl->m_Arena) const Span*[SpanCount];
+
+ uint64_t Index = 0;
+
+ for (Span* CurrentSpan = m_Impl->m_SpanChain; CurrentSpan != nullptr; CurrentSpan = CurrentSpan->m_SpanChain)
+ {
+ Spans[Index++] = CurrentSpan;
+ }
+
+ Recorder->RecordSpans(t_TraceState.CurrentTraceId, std::span<const Span*>(Spans, SpanCount));
+ }
+
+ delete m_Impl;
+}
+
+Tracer*
+Tracer::GetTracer()
+{
+ Tracer* TracerPtr = t_Tracer;
+
+ if (!TracerPtr)
+ {
+ TracerPtr = new Tracer();
+ }
+
+ return TracerPtr;
+}
+
+ScopedSpan
+Tracer::CreateSpan(std::string_view Name)
+{
+ Tracer* TracerPtr = GetTracer();
+
+ Impl* const ImplPtr = TracerPtr->m_Impl;
+
+ Span* NewSpan = new (ImplPtr->m_Arena) Span(ImplPtr->m_Arena, Name, ImplPtr->m_SpanChain);
+
+ ImplPtr->m_SpanChain = NewSpan;
+ ImplPtr->m_SpanCount.fetch_add(1);
+
+ return ScopedSpan(NewSpan, TracerPtr);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+ScopedSpan::ScopedSpan(std::string_view Name)
+{
+ Tracer* TracerPtr = Tracer::GetTracer();
+
+ Tracer::Impl* const ImplPtr = TracerPtr->m_Impl;
+
+ Span* NewSpan = new (ImplPtr->m_Arena) Span(ImplPtr->m_Arena, Name, ImplPtr->m_SpanChain);
+
+ ImplPtr->m_SpanChain = NewSpan;
+ ImplPtr->m_SpanCount.fetch_add(1);
+
+ m_Tracer = TracerPtr;
+ m_Span = NewSpan;
+}
+
+ScopedSpan::ScopedSpan(Span* InSpan, Tracer* InTracer) : m_Tracer(InTracer), m_Span(InSpan)
+{
+}
+
+ScopedSpan::~ScopedSpan()
+{
+}
+
+} // namespace zen::otel
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace zen::otel {
+
+using namespace std::literals;
+
+void
+otlptrace_forcelink()
+{
+}
+
+# if ZEN_WITH_TESTS
+
+TEST_CASE("otlp.trace")
+{
+ {
+ ScopedSpan Span = Tracer::CreateSpan("span0");
+ Span->AddEvent("TestEvent1"sv);
+ Span->AddEvent("TestEvent2"sv);
+
+ {
+ ScopedSpan Span2 = Tracer::CreateSpan("span1");
+ Span2->AddEvent("TestEvent3"sv);
+ Span2->AddEvent("TestEvent4"sv);
+ }
+ }
+}
+
+# endif
+
+} // namespace zen::otel
+#endif
diff --git a/src/zencore/stats.cpp b/src/zentelemetry/stats.cpp
index 8a424c5ad..c67fa3c66 100644
--- a/src/zencore/stats.cpp
+++ b/src/zentelemetry/stats.cpp
@@ -1,6 +1,6 @@
// Copyright Epic Games, Inc. All Rights Reserved.
-#include "zencore/stats.h"
+#include "zentelemetry/stats.h"
#include <zencore/compactbinarybuilder.h>
#include <zencore/intmath.h>
diff --git a/src/zentelemetry/xmake.lua b/src/zentelemetry/xmake.lua
new file mode 100644
index 000000000..2f90d7b90
--- /dev/null
+++ b/src/zentelemetry/xmake.lua
@@ -0,0 +1,10 @@
+-- Copyright Epic Games, Inc. All Rights Reserved.
+
+target('zentelemetry')
+ set_kind("static")
+ set_group("libs")
+ add_headerfiles("**.h")
+ add_files("**.cpp")
+ add_includedirs("include", {public=true})
+ add_deps("zencore", "protozero")
+ add_packages("vcpkg::robin-map", "vcpkg::spdlog")
diff --git a/src/zentelemetry/zentelemetry.cpp b/src/zentelemetry/zentelemetry.cpp
new file mode 100644
index 000000000..ed6ad13b9
--- /dev/null
+++ b/src/zentelemetry/zentelemetry.cpp
@@ -0,0 +1,19 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zentelemetry/zentelemetry.h"
+
+#include "zentelemetry/otlptrace.h"
+#include "zentelemetry/stats.h"
+
+namespace zen {
+
+void
+zentelemetry_forcelinktests()
+{
+ zen::stats_forcelink();
+#if ZEN_WITH_OTEL
+ zen::otel::otlptrace_forcelink();
+#endif
+}
+
+} // namespace zen
diff --git a/thirdparty/protozero/CHANGELOG.md b/thirdparty/protozero/CHANGELOG.md
new file mode 100644
index 000000000..12231fe4d
--- /dev/null
+++ b/thirdparty/protozero/CHANGELOG.md
@@ -0,0 +1,457 @@
+
+# Changelog
+
+All notable changes to this project will be documented in this file.
+The format is based on [Keep a Changelog](https://keepachangelog.com/)
+This project adheres to [Semantic Versioning](https://semver.org/).
+
+## [unreleased] -
+
+### Added
+
+### Changed
+
+### Fixed
+
+
+## [1.8.1] - 2025-07-15
+
+### Fixed
+
+- Fix buffer overrun in `get_bool()`
+- Fix test that checks that protozero also works with `std::string_view`
+
+
+## [1.8.0] - 2025-01-13
+
+### Changed
+
+- Switched to C++14 as minimum requirement. Modernised the code accordingly.
+- Only the tests need the Protobuf library when building.
+- Improve compatibility with various STL flavors.
+- Change license formatting to get recognized by license scanning tools.
+- Add lots of `const` in places where we can. Also fix some other issues
+ reported by clang-tidy.
+- Various other code cleanups.
+- Modernize documentation config, CMake config and CI workflows.
+
+### Fixed
+
+- `basic_pbf_builder` constructor can not be noexcept.
+
+
+## [1.7.1] - 2022-01-10
+
+### Changed
+
+- Don't build tests if the standard CMake `BUILD_TESTING` variable is set to
+ off.
+- Now needs CMake 3.5.0 or greater.
+- Update included catch2 framework to current version v2.13.8.
+- Only enable clang-tidy make target if protobuf was found.
+- Allow setting C++ version to compile with in CMake config.
+
+### Fixed
+
+- Fixes undefined behaviour in `float` and `double` byteswap.
+- Add missing includes of "config.hpp".
+- Avoid narrowing conversion by doing an explicit `static_cast`.
+
+
+## [1.7.0] - 2020-06-08
+
+### Added
+
+- Support for buffer types other that `std::string`. `pbf_writer` is now
+ just a typedef for `basic_pbf_writer<std::string>`. Other buffer types
+ can be used with `basic_pbf_writer`. See `doc/advanced.md` for details.
+
+### Changed
+
+- Switched to *catch2* for testing.
+- Some minor tweaks.
+
+### Fixed
+
+- Removed some undefined behaviour.
+
+
+## [1.6.8] - 2019-08-15
+
+### Changed
+
+- Various code cleanups due to clang-tidy warnings.
+
+### Fixed
+
+- Made `data_view::compare` noexcept.
+
+
+## [1.6.7] - 2018-02-21
+
+### Fixed
+
+- Signed-unsigned comparison on 32 bit systems.
+
+
+## [1.6.6] - 2018-02-20
+
+### Fixed
+
+- Fixed several place with possible undefined behaviour.
+
+
+## [1.6.5] - 2018-02-05
+
+### Fixed
+
+- Avoid UB: Do not calculate pointer outside array bounds.
+- Specify proto2 syntax in .proto files to appease protoc.
+
+
+## [1.6.4] - 2018-11-08
+
+### Added
+
+- Add function `data()` to get the not yet read data from a `pbf_reader`.
+- New `add_packed_fixed()` template function for `pbf_writer`.
+- New `length_of_varint()` helper function calculates how long a varint
+ would be for a specified value.
+
+### Changed
+
+- More consistent implementation of operators as free friend functions.
+
+### Fixed
+
+- Fixed some zigzag encoding tests on MSVC.
+- Add extra cast so we do an xor with unsigned ints.
+- No more bitwise operations on signed integers in varint decoder.
+- No more bitwise operations on signed integers in zigzag encoder/decoder.
+
+
+## [1.6.3] - 2018-07-17
+
+### Changed
+
+- Moved `byteswap_inplace` functions from detail into protozero namespace.
+ They can be useful outsize protozero.
+- More asserts and unit tests and small cleanups.
+
+
+## [1.6.2] - 2018-03-09
+
+### Changed
+
+- Update included catch.hpp to v1.12.0.
+- Move basic unit tests into their own directory (`test/unit`).
+- Improved clang-tidy config and fixed some code producing warnings.
+
+### Fixed
+
+- Buffer overflow in pbf-decoder tool.
+
+
+## [1.6.1] - 2017-11-16
+
+### Added
+
+- Document internal handling of varints.
+- Add aliases for fixed iterators, too.
+
+### Changed
+
+- The `const_fixed_iterator` is now a random access iterator making code
+ using it potentially more performant (for instance when using
+ `std::distance`)
+- Overloads `std::distance` for the varint and svarint iterators. This is
+ better than the workaround with the `rage_size` function used before.
+
+### Fixed
+
+- Rename `.proto` files in some tests to be unique. This solves a problem
+ when building with newer versions of the Google Protobuf library.
+- Floating point comparisons in tests are now always correctly done using
+ `Approx()`.
+
+
+## [1.6.0] - 2017-10-24
+
+### Added
+
+- Comparison functions (<, <=, >, >=) for `data_view`. Allows use in `std::map`
+ for instance.
+- Tool `pbf-decoder` for decoding raw messages. This has limited use for
+ normal users, but it can be used for fuzzing.
+
+### Changed
+
+- Protozero now uses CMake to build the tests etc. This does not affect
+ simple users of the library, but if you are using CMake yourself you might
+ want to use the `cmake/FindProtozero.cmake` module provided. The README
+ contains more information about build options.
+- Moved `data_view` class from `types.hpp` into its own header file
+ `data_view.hpp`.
+- Implementation of the `const_fixed_iterator` to use only a single pointer
+ instead of two.
+- Made `operator==` and `operator!=` on `data_view` constexpr.
+- The `pbf_reader` constructor taking a `std::pair` is deprecated. Use one
+ of the other constructors instead.
+
+### Fixed
+
+- Varints where the last byte was larger than what would fit in 64bit were
+ triggering undefined behaviour. This can only happen when the message
+ being decoded was corrupt in some way.
+- Do not assert when reading too long varints for bools any more. A valid
+ encoder should never generate varints with more than one byte for bools,
+ but if they are longer that's not really a problem, so just handle it.
+- Throw exception if the length of a packed repeated field of a fixed-length
+ type is invalid. The length must always be a multiple of the size of the
+ underlying type. This can only happen if the data is corrupted in some way,
+ a valid encoder would never generate data like this.
+- Throw an exception when reading invalid tags. This can only happen if the
+ data is corrupted in some way, a valid encoder would never generate invalid
+ tags.
+
+
+## [1.5.3] - 2017-09-22
+
+### Added
+
+- More documentation.
+- New `size()` method on iterator range used for packed repeated fields to
+ find out how many elements there are in the range. This is much faster
+ compared to the `std::difference()` call you had to do before, because the
+ varints don't have to be fully decoded. See [Advanced
+ Topics](doc/advanced.md) for details.
+
+### Changed
+
+- Updated clang-tidy settings in Makefiles and fixed a lot of minor issues
+ reported by clang-tidy.
+- Update included catch.hpp to version 1.10.0.
+- Miscellaneous code cleanups.
+- Support for invalid state in `pbf_writer` and `packed_repeated_fields`.
+ This fixes move construction and move assignement in `pbf_writer` and
+ disables the copy construction and copy assignement which don't have
+ clear semantics. It introduces an invalid or empty state in the
+ `pbf_writer`, `pbf_builder`, and `packed_repeated_fields` classes used for
+ default-constructed, moved from, or committed objects. There is a new
+ `commit()` function for `pbf_writer` and the `packed_repeated_fields` which
+ basically does the same as the destructor but can be called explicitly.
+
+### Fixed
+
+- The `empty()` method of the iterator range now returns a `bool` instead of
+ a `size_t`.
+
+
+## [1.5.2] - 2017-06-30
+
+### Added
+
+- Add missing two-parameter version of `pbf_message::next()` function.
+- Add `data_view::empty()` function.
+- Add missing versions of `add_bytes()`, `add_string()`, and `add_message()`
+ to `pbf_builder`.
+
+### Changed
+
+- Clarify include file usage in tutorial.
+- Updated included Catch unit test framework to version 1.9.6 and updated
+ tests to work with the current version.
+- Make some constructors explicit (best practice to avoid silent conversions).
+
+### Fixed
+
+- Important bugfix in `data_view` equality operator. The equality operator is
+ actually never used in the protozero code itself, but users of protozero
+ might use it. This is a serious bug that could lead to buffer overrun type
+ problems.
+
+
+## [1.5.1] - 2017-01-14
+
+### Added
+
+- Better documentation for `tag_and_type()` in doc/advanced.md.
+
+### Fixed
+
+- Fixed broken "make doc" build.
+
+
+## [1.5.0] - 2017-01-12
+
+### Added
+
+- Add `add_bytes_vectored()` methods to `pbf_writer` and `pbf_builder`. This
+ allows single-copy scatter-gather type adding of data that has been prepared
+ in pieces to a protobuf message.
+- New functions to check the tag and wire type at the same time: Two parameter
+ version of `pbf_reader::next()` and `pbf_reader::tag_and_type()` can be used
+ together with the free function `tag_and_type()` to easily and quickly check
+ that not only the tag but also the wire type is correct for a field.
+
+### Changed
+
+- `packed_field_*` classes now work with `pbf_builder`.
+- Reorganized documentation. Advanced docs are now under doc/advanced.md.
+
+### Fixed
+
+- `packed_field` class is now non-copyable because data can get corrupted if
+ you copy it around.
+- Comparison operators of `data_view` now have const& parameters.
+- Make zigzag encoding/decoding functions constexpr.
+
+
+## [1.4.5] - 2016-11-18
+
+### Fixed
+
+- Undefined behaviour in packed fixed iterator. As a result, the macro
+ `PROTOZERO_DO_NOT_USE_BARE_POINTER` is not used any more.
+
+
+## [1.4.4] - 2016-11-15
+
+### Fixed
+
+- Byteswap implementation.
+
+
+## [1.4.3] - 2016-11-15
+
+### Fixed
+
+- Undefined behaviour in byte swapping code.
+- Rename some parameters to avoid "shadow" warning from some compilers.
+
+
+## [1.4.2] - 2016-08-27
+
+### Fixed
+
+- Compile fix: Variable shadowing.
+
+
+## [1.4.1] - 2016-08-21
+
+### Fixed
+
+- GCC 4.8 compile fixed
+
+### Added
+
+- New ability to dynamically require the module as a node module to ease
+ building against from other node C++ modules.
+
+## [1.4.0] - 2016-07-22
+
+### Changed
+
+- Use more efficient new `skip_varint()` function when iterating over
+ packed varints.
+- Split `decode_varint()` function into two functions speeding up the
+ common case where a varint is only one byte long.
+- Introduce new class `iterator_range` used instead of `std::pair` of
+ iterators. This way the objects can be used in range-based for loops.
+ Read UPGRADING.md for details.
+- Introduce new class `data_view` and functions using and returning it.
+ Read UPGRADING.md for details.
+
+
+## [1.3.0] - 2016-02-18
+
+### Added
+
+- Added `config.hpp` header which now includes all the macro magic to
+ configure the library for different architectures etc.
+- New way to create repeated packed fields without using an iterator.
+- Add `rollback()` function to `pbf_writer` for "manual" rollback.
+
+### Changed
+
+- Various test and documentation cleanups.
+- Rename `pbf_types.hpp` to `types.hpp`.
+
+
+## [1.2.3] - 2015-11-30
+
+### Added
+
+- Added `config.hpp` header which now includes all the macro magic to
+ configure the library for different architectures etc.
+
+### Fixed
+
+- Unaligned access to floats/doubles on some ARM architectures.
+
+
+## [1.2.2] - 2015-10-13
+
+### Fixed
+
+- Fix the recently broken writing of bools on big-endian architectures.
+
+
+## [1.2.1] - 2015-10-12
+
+### Fixed
+
+- Removed unneeded code (1-byte "swap") which lead to test failures.
+
+
+## [1.2.0] - 2015-10-08
+
+### Added
+
+- `pbf_message` and `pbf_builder` template classes wrapping `pbf_reader`
+ and `pbf_writer`, respectively. The new classes are the preferred
+ interface now.
+
+### Changed
+
+- Improved byte swapping operation.
+- Detect some types of data corruption earlier and throw.
+
+
+## [1.1.0] - 2015-08-22
+
+### Changed
+
+- Make pbf reader and writer code endianess-aware.
+
+
+[unreleased]: https://github.com/osmcode/libosmium/compare/v1.8.0...HEAD
+[1.8.0]: https://github.com/osmcode/libosmium/compare/v1.7.1...v1.8.0
+[1.7.1]: https://github.com/osmcode/libosmium/compare/v1.7.0...v1.7.1
+[1.7.0]: https://github.com/osmcode/libosmium/compare/v1.6.8...v1.7.0
+[1.6.8]: https://github.com/osmcode/libosmium/compare/v1.6.7...v1.6.8
+[1.6.7]: https://github.com/osmcode/libosmium/compare/v1.6.6...v1.6.7
+[1.6.6]: https://github.com/osmcode/libosmium/compare/v1.6.5...v1.6.6
+[1.6.5]: https://github.com/osmcode/libosmium/compare/v1.6.4...v1.6.5
+[1.6.4]: https://github.com/osmcode/libosmium/compare/v1.6.3...v1.6.4
+[1.6.3]: https://github.com/osmcode/libosmium/compare/v1.6.2...v1.6.3
+[1.6.2]: https://github.com/osmcode/libosmium/compare/v1.6.1...v1.6.2
+[1.6.1]: https://github.com/osmcode/libosmium/compare/v1.6.0...v1.6.1
+[1.6.0]: https://github.com/osmcode/libosmium/compare/v1.5.3...v1.6.0
+[1.5.3]: https://github.com/osmcode/libosmium/compare/v1.5.2...v1.5.3
+[1.5.2]: https://github.com/osmcode/libosmium/compare/v1.5.1...v1.5.2
+[1.5.1]: https://github.com/osmcode/libosmium/compare/v1.5.0...v1.5.1
+[1.5.0]: https://github.com/osmcode/libosmium/compare/v1.4.5...v1.5.0
+[1.4.5]: https://github.com/osmcode/libosmium/compare/v1.4.4...v1.4.5
+[1.4.4]: https://github.com/osmcode/libosmium/compare/v1.4.3...v1.4.4
+[1.4.3]: https://github.com/osmcode/libosmium/compare/v1.4.2...v1.4.3
+[1.4.2]: https://github.com/osmcode/libosmium/compare/v1.4.1...v1.4.2
+[1.4.1]: https://github.com/osmcode/libosmium/compare/v1.4.0...v1.4.1
+[1.4.0]: https://github.com/osmcode/libosmium/compare/v1.3.0...v1.4.0
+[1.3.0]: https://github.com/osmcode/libosmium/compare/v1.2.3...v1.3.0
+[1.2.3]: https://github.com/osmcode/libosmium/compare/v1.2.2...v1.2.3
+[1.2.2]: https://github.com/osmcode/libosmium/compare/v1.2.1...v1.2.2
+[1.2.1]: https://github.com/osmcode/libosmium/compare/v1.2.0...v1.2.1
+[1.2.0]: https://github.com/osmcode/libosmium/compare/v1.1.0...v1.2.0
+[1.1.0]: https://github.com/osmcode/libosmium/compare/v1.0.0...v1.1.0
+
diff --git a/thirdparty/protozero/LICENSE.from_folly b/thirdparty/protozero/LICENSE.from_folly
new file mode 100644
index 000000000..f433b1a53
--- /dev/null
+++ b/thirdparty/protozero/LICENSE.from_folly
@@ -0,0 +1,177 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
diff --git a/thirdparty/protozero/LICENSE.md b/thirdparty/protozero/LICENSE.md
new file mode 100644
index 000000000..5c2ce01d1
--- /dev/null
+++ b/thirdparty/protozero/LICENSE.md
@@ -0,0 +1,26 @@
+Copyright (C) 2022, Mapbox.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/thirdparty/protozero/README.md b/thirdparty/protozero/README.md
new file mode 100644
index 000000000..a97ef3fc0
--- /dev/null
+++ b/thirdparty/protozero/README.md
@@ -0,0 +1,155 @@
+# protozero
+
+Minimalistic protocol buffer decoder and encoder in C++.
+
+Designed for high performance. Suitable for writing zero copy parsers and
+encoders with minimal need for run-time allocation of memory.
+
+Low-level: this is designed to be a building block for writing a very
+customized decoder for a stable protobuf schema. If your protobuf schema is
+changing frequently or lazy decoding is not critical for your application then
+this approach offers no value: just use the C++ API that can be generated with
+the Google Protobufs `protoc` program.
+
+[![Github Build Status](https://github.com/mapbox/protozero/actions/workflows/ci.yml/badge.svg)](https://github.com/mapbox/protozero/actions/workflows/ci.yml)
+[![Appveyor Build Status](https://ci.appveyor.com/api/projects/status/github/mapbox/protozero?svg=true)](https://ci.appveyor.com/project/Mapbox/protozero)
+[![Packaging status](https://repology.org/badge/tiny-repos/protozero.svg)](https://repology.org/metapackage/protozero)
+
+## Depends
+
+* C++14 compiler
+* CMake
+* Some tests depend on the Google Protobuf library, but use of Protozero
+ doesn't need it
+
+
+## How it works
+
+The protozero code does **not** read `.proto` files used by the usual Protobuf
+implementations. The developer using protozero has to manually "translate" the
+`.proto` description into code. This means there is no way to access any of the
+information from the `.proto` description. This results in a few restrictions:
+
+* The names of the fields are not available.
+* Enum names are not available, you'll have to use the values they are defined
+ with.
+* Default values are not available.
+* Field types have to be hardcoded. The library does not know which types to
+ expect, so the user of the library has to supply the right types. Some checks
+ are made using `assert()`, but mostly the user has to take care of that.
+
+The library will make sure not to overrun the buffer it was given, but
+basically all other checks have to be made in user code!
+
+
+## Documentation
+
+You have to have a working knowledge of how
+[protocol buffer encoding works](https://developers.google.com/protocol-buffers/docs/encoding).
+
+* Read the [tutorial](doc/tutorial.md) for an introduction on how to use
+ Protozero.
+* Some advanced topics are described in an [extra document](doc/advanced.md).
+* There is a table of all types and functions in the
+ [cheat sheet](doc/cheatsheet.md).
+* Read the [upgrading instructions](UPGRADING.md) if you are upgrading from
+ an older version of Protozero.
+
+The build process will also build the Doxygen-based reference documentation if
+you have Doxygen installed. Then open `doc/html/index.html` in your browser to
+read it.
+
+
+## Endianness
+
+Protozero uses a very simplistic test to check the byte order of the system it
+compiles on. If this check is wrong, you'll get test failures. If this is the
+case, please [open an issue](https://github.com/mapbox/protozero/issues) and
+tell us about your system.
+
+
+## Building tests
+
+Extensive tests are included. Build them using CMake:
+
+ mkdir build
+ cd build
+ cmake ..
+ make
+
+Call `ctest` to run the tests.
+
+The unit and reader tests are always build, the writer tests are only build if
+the Google Protobuf library is found when running CMake.
+
+See `test/README.md` for more details about the test.
+
+
+## Coverage report
+
+To get a coverage report set `CXXFLAGS` and `LDFLAGS` before calling CMake:
+
+ CXXFLAGS="--coverage" LDFLAGS="--coverage" cmake ..
+
+Then call `make` as usual and run the tests using `ctest`.
+
+If you are using `g++` use `gcov` to generate a report (results are in `*.gcov`
+files):
+
+ gcov -lp $(find test/ -name '*.o')
+
+If you are using `clang++` use `llvm-cov` instead:
+
+ llvm-cov gcov -lp $(find test/ -name '*.o')
+
+If you are using `g++` you can use `gcovr` to generate nice HTML output:
+
+ mkdir -p coverage
+ gcovr . -r SRCDIR --html --html-details -o coverage/index.html
+
+Open `coverage/index.html` in your browser to see the report.
+
+
+## Clang-tidy
+
+After the CMake step, run
+
+ make clang-tidy
+
+to check the code with [clang-tidy](https://clang.llvm.org/extra/clang-tidy/).
+You might have to set `CLANG_TIDY` in CMake config.
+
+
+## Cppcheck
+
+For extra checks with [Cppcheck](https://cppcheck.sourceforge.io/) you can,
+after the CMake step, call
+
+ make cppcheck
+
+
+## Installation
+
+After the CMake step, call `make install` to install the include files in
+`/usr/local/include/protozero`.
+
+If you are using CMake to build your own software, you can copy the file
+`cmake/FindProtozero.cmake` and use it in your build. See the file for
+details.
+
+
+## Who is using Protozero?
+
+* [Carmen](https://github.com/mapbox/carmen-cache)
+* [Libosmium](https://github.com/osmcode/libosmium)
+* [Mapbox GL Native](https://github.com/mapbox/mapbox-gl-native)
+* [Mapbox Vector Tile library](https://github.com/mapbox/vector-tile)
+* [Mapnik](https://github.com/mapbox/mapnik-vector-tile)
+* [OSRM](https://github.com/Project-OSRM/osrm-backend)
+* [Tippecanoe](https://github.com/mapbox/tippecanoe)
+* [Vtzero](https://github.com/mapbox/vtzero)
+
+Are you using Protozero? Tell us! Send a pull request with changes to this
+README.
+
+
diff --git a/thirdparty/protozero/include/protozero/basic_pbf_builder.hpp b/thirdparty/protozero/include/protozero/basic_pbf_builder.hpp
new file mode 100644
index 000000000..ce7640d43
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/basic_pbf_builder.hpp
@@ -0,0 +1,266 @@
+#ifndef PROTOZERO_BASIC_PBF_BUILDER_HPP
+#define PROTOZERO_BASIC_PBF_BUILDER_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file basic_pbf_builder.hpp
+ *
+ * @brief Contains the basic_pbf_builder template class.
+ */
+
+#include "basic_pbf_writer.hpp"
+#include "types.hpp"
+
+#include <type_traits>
+
+namespace protozero {
+
+/**
+ * The basic_pbf_builder is used to write PBF formatted messages into a buffer.
+ * It is based on the basic_pbf_writer class and has all the same methods. The
+ * difference is that while the pbf_writer class takes an integer tag,
+ * this template class takes a tag of the template type T. The idea is that
+ * T will be an enumeration value and this helps reduce the possibility of
+ * programming errors.
+ *
+ * Almost all methods in this class can throw an std::bad_alloc exception if
+ * the underlying buffer class wants to resize.
+ *
+ * Read the tutorial to understand how this class is used. In most cases you
+ * want to use the pbf_builder class which uses a std::string as buffer type.
+ */
+template <typename TBuffer, typename T>
+class basic_pbf_builder : public basic_pbf_writer<TBuffer> {
+
+ static_assert(std::is_same<pbf_tag_type, std::underlying_type_t<T>>::value,
+ "T must be enum with underlying type protozero::pbf_tag_type");
+
+public:
+
+ /// The type of messages this class will build.
+ using enum_type = T;
+
+ basic_pbf_builder() = default;
+
+ /**
+ * Create a builder using the given string as a data store. The object
+ * stores a reference to that string and adds all data to it. The string
+ * doesn't have to be empty. The pbf_message object will just append data.
+ */
+ explicit basic_pbf_builder(TBuffer& data) noexcept :
+ basic_pbf_writer<TBuffer>{data} {
+ }
+
+ /**
+ * Construct a pbf_builder for a submessage from the pbf_message or
+ * pbf_writer of the parent message.
+ *
+ * @param parent_writer The parent pbf_message or pbf_writer
+ * @param tag Tag of the field that will be written
+ */
+ template <typename P>
+ basic_pbf_builder(basic_pbf_writer<TBuffer>& parent_writer, P tag) :
+ basic_pbf_writer<TBuffer>{parent_writer, pbf_tag_type(tag)} {
+ }
+
+/// @cond INTERNAL
+#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \
+ void add_##name(T tag, type value) { \
+ basic_pbf_writer<TBuffer>::add_##name(pbf_tag_type(tag), value); \
+ }
+
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float)
+ PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double)
+
+#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR
+/// @endcond
+
+ /**
+ * Add "bytes" field to data.
+ *
+ * @param tag Tag of the field
+ * @param value Pointer to value to be written
+ * @param size Number of bytes to be written
+ */
+ void add_bytes(T tag, const char* value, std::size_t size) {
+ basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value, size);
+ }
+
+ /**
+ * Add "bytes" field to data.
+ *
+ * @param tag Tag of the field
+ * @param value Value to be written
+ */
+ void add_bytes(T tag, const data_view& value) {
+ basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value);
+ }
+
+ /**
+ * Add "bytes" field to data.
+ *
+ * @param tag Tag of the field
+ * @param value Value to be written
+ */
+ void add_bytes(T tag, const std::string& value) {
+ basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value);
+ }
+
+ /**
+ * Add "bytes" field to data. Bytes from the value are written until
+ * a null byte is encountered. The null byte is not added.
+ *
+ * @param tag Tag of the field
+ * @param value Pointer to zero-delimited value to be written
+ */
+ void add_bytes(T tag, const char* value) {
+ basic_pbf_writer<TBuffer>::add_bytes(pbf_tag_type(tag), value);
+ }
+
+ /**
+ * Add "bytes" field to data using vectored input. All the data in the
+ * 2nd and further arguments is "concatenated" with only a single copy
+ * into the final buffer.
+ *
+ * This will work with objects of any type supporting the data() and
+ * size() methods like std::string or protozero::data_view.
+ *
+ * Example:
+ * @code
+ * std::string data1 = "abc";
+ * std::string data2 = "xyz";
+ * builder.add_bytes_vectored(1, data1, data2);
+ * @endcode
+ *
+ * @tparam Ts List of types supporting data() and size() methods.
+ * @param tag Tag of the field
+ * @param values List of objects of types Ts with data to be appended.
+ */
+ template <typename... Ts>
+ void add_bytes_vectored(T tag, Ts&&... values) {
+ basic_pbf_writer<TBuffer>::add_bytes_vectored(pbf_tag_type(tag), std::forward<Ts>(values)...);
+ }
+
+ /**
+ * Add "string" field to data.
+ *
+ * @param tag Tag of the field
+ * @param value Pointer to value to be written
+ * @param size Number of bytes to be written
+ */
+ void add_string(T tag, const char* value, std::size_t size) {
+ basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value, size);
+ }
+
+ /**
+ * Add "string" field to data.
+ *
+ * @param tag Tag of the field
+ * @param value Value to be written
+ */
+ void add_string(T tag, const data_view& value) {
+ basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value);
+ }
+
+ /**
+ * Add "string" field to data.
+ *
+ * @param tag Tag of the field
+ * @param value Value to be written
+ */
+ void add_string(T tag, const std::string& value) {
+ basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value);
+ }
+
+ /**
+ * Add "string" field to data. Bytes from the value are written until
+ * a null byte is encountered. The null byte is not added.
+ *
+ * @param tag Tag of the field
+ * @param value Pointer to value to be written
+ */
+ void add_string(T tag, const char* value) {
+ basic_pbf_writer<TBuffer>::add_string(pbf_tag_type(tag), value);
+ }
+
+ /**
+ * Add "message" field to data.
+ *
+ * @param tag Tag of the field
+ * @param value Pointer to message to be written
+ * @param size Length of the message
+ */
+ void add_message(T tag, const char* value, std::size_t size) {
+ basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value, size);
+ }
+
+ /**
+ * Add "message" field to data.
+ *
+ * @param tag Tag of the field
+ * @param value Value to be written. The value must be a complete message.
+ */
+ void add_message(T tag, const data_view& value) {
+ basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value);
+ }
+
+ /**
+ * Add "message" field to data.
+ *
+ * @param tag Tag of the field
+ * @param value Value to be written. The value must be a complete message.
+ */
+ void add_message(T tag, const std::string& value) {
+ basic_pbf_writer<TBuffer>::add_message(pbf_tag_type(tag), value);
+ }
+
+/// @cond INTERNAL
+#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \
+ template <typename InputIterator> \
+ void add_packed_##name(T tag, InputIterator first, InputIterator last) { \
+ basic_pbf_writer<TBuffer>::add_packed_##name(pbf_tag_type(tag), first, last); \
+ }
+
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(bool)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(enum)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(int32)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(int64)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(float)
+ PROTOZERO_WRITER_WRAP_ADD_PACKED(double)
+
+#undef PROTOZERO_WRITER_WRAP_ADD_PACKED
+/// @endcond
+
+}; // class basic_pbf_builder
+
+} // end namespace protozero
+
+#endif // PROTOZERO_BASIC_PBF_BUILDER_HPP
diff --git a/thirdparty/protozero/include/protozero/basic_pbf_writer.hpp b/thirdparty/protozero/include/protozero/basic_pbf_writer.hpp
new file mode 100644
index 000000000..c26ba67cd
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/basic_pbf_writer.hpp
@@ -0,0 +1,1054 @@
+#ifndef PROTOZERO_BASIC_PBF_WRITER_HPP
+#define PROTOZERO_BASIC_PBF_WRITER_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file basic_pbf_writer.hpp
+ *
+ * @brief Contains the basic_pbf_writer template class.
+ */
+
+#include "buffer_tmpl.hpp"
+#include "config.hpp"
+#include "data_view.hpp"
+#include "types.hpp"
+#include "varint.hpp"
+
+#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
+# include <protozero/byteswap.hpp>
+#endif
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <initializer_list>
+#include <iterator>
+#include <limits>
+#include <string>
+#include <utility>
+
+namespace protozero {
+
+namespace detail {
+
+ template <typename B, typename T> class packed_field_varint;
+ template <typename B, typename T> class packed_field_svarint;
+ template <typename B, typename T> class packed_field_fixed;
+
+} // end namespace detail
+
+/**
+ * The basic_pbf_writer is used to write PBF formatted messages into a buffer.
+ *
+ * This uses TBuffer as the type for the underlaying buffer. In typical uses
+ * this is std::string, but you can use a different type that must support
+ * the right interface. Please see the documentation for details.
+ *
+ * Almost all methods in this class can throw an std::bad_alloc exception if
+ * the underlying buffer class wants to resize.
+ */
+template <typename TBuffer>
+class basic_pbf_writer {
+
+ // A pointer to a buffer holding the data already written to the PBF
+ // message. For default constructed writers or writers that have been
+ // rolled back, this is a nullptr.
+ TBuffer* m_data = nullptr;
+
+ // A pointer to a parent writer object if this is a submessage. If this
+ // is a top-level writer, it is a nullptr.
+ basic_pbf_writer* m_parent_writer = nullptr;
+
+ // This is usually 0. If there is an open submessage, this is set in the
+ // parent to the rollback position, ie. the last position before the
+ // submessage was started. This is the position where the header of the
+ // submessage starts.
+ std::size_t m_rollback_pos = 0;
+
+ // This is usually 0. If there is an open submessage, this is set in the
+ // parent to the position where the data of the submessage is written to.
+ std::size_t m_pos = 0;
+
+ void add_varint(uint64_t value) {
+ protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage");
+ protozero_assert(m_data);
+ add_varint_to_buffer(m_data, value);
+ }
+
+ void add_field(pbf_tag_type tag, pbf_wire_type type) {
+ protozero_assert(((tag > 0 && tag < 19000) || (tag > 19999 && tag <= ((1U << 29U) - 1))) && "tag out of range");
+ const uint32_t b = (tag << 3U) | static_cast<uint32_t>(type);
+ add_varint(b);
+ }
+
+ void add_tagged_varint(pbf_tag_type tag, uint64_t value) {
+ add_field(tag, pbf_wire_type::varint);
+ add_varint(value);
+ }
+
+ template <typename T>
+ void add_fixed(T value) {
+ protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage");
+ protozero_assert(m_data);
+#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
+ byteswap_inplace(&value);
+#endif
+ buffer_customization<TBuffer>::append(m_data, reinterpret_cast<const char*>(&value), sizeof(T));
+ }
+
+ template <typename T, typename It>
+ void add_packed_fixed(pbf_tag_type tag, It first, It last, std::input_iterator_tag /*unused*/) { // NOLINT(performance-unnecessary-value-param)
+ if (first == last) {
+ return;
+ }
+
+ basic_pbf_writer sw{*this, tag};
+
+ while (first != last) {
+ sw.add_fixed<T>(*first++);
+ }
+ }
+
+ template <typename T, typename It>
+ void add_packed_fixed(pbf_tag_type tag, It first, It last, std::forward_iterator_tag /*unused*/) { // NOLINT(performance-unnecessary-value-param)
+ if (first == last) {
+ return;
+ }
+
+ const auto length = std::distance(first, last);
+ add_length_varint(tag, sizeof(T) * pbf_length_type(length));
+ reserve(sizeof(T) * std::size_t(length));
+
+ while (first != last) {
+ add_fixed<T>(*first++);
+ }
+ }
+
+ template <typename It>
+ void add_packed_varint(pbf_tag_type tag, It first, It last) { // NOLINT(performance-unnecessary-value-param)
+ if (first == last) {
+ return;
+ }
+
+ basic_pbf_writer sw{*this, tag};
+
+ while (first != last) {
+ sw.add_varint(uint64_t(*first++));
+ }
+ }
+
+ template <typename It>
+ void add_packed_svarint(pbf_tag_type tag, It first, It last) { // NOLINT(performance-unnecessary-value-param)
+ if (first == last) {
+ return;
+ }
+
+ basic_pbf_writer sw{*this, tag};
+
+ while (first != last) {
+ sw.add_varint(encode_zigzag64(*first++));
+ }
+ }
+
+ // The number of bytes to reserve for the varint holding the length of
+ // a length-delimited field. The length has to fit into pbf_length_type,
+ // and a varint needs 8 bit for every 7 bit.
+ enum : int {
+ reserve_bytes = (sizeof(pbf_length_type) * 8 / 7) + 1
+ };
+
+ // If m_rollpack_pos is set to this special value, it means that when
+ // the submessage is closed, nothing needs to be done, because the length
+ // of the submessage has already been written correctly.
+ enum : std::size_t {
+ size_is_known = std::numeric_limits<std::size_t>::max()
+ };
+
+ void open_submessage(pbf_tag_type tag, std::size_t size) {
+ protozero_assert(m_pos == 0);
+ protozero_assert(m_data);
+ if (size == 0) {
+ m_rollback_pos = buffer_customization<TBuffer>::size(m_data);
+ add_field(tag, pbf_wire_type::length_delimited);
+ buffer_customization<TBuffer>::append_zeros(m_data, std::size_t(reserve_bytes));
+ } else {
+ m_rollback_pos = size_is_known;
+ add_length_varint(tag, static_cast<pbf_length_type>(size));
+ reserve(size);
+ }
+ m_pos = buffer_customization<TBuffer>::size(m_data);
+ }
+
+ void rollback_submessage() {
+ protozero_assert(m_pos != 0);
+ protozero_assert(m_rollback_pos != size_is_known);
+ protozero_assert(m_data);
+ buffer_customization<TBuffer>::resize(m_data, m_rollback_pos);
+ m_pos = 0;
+ }
+
+ void commit_submessage() {
+ protozero_assert(m_pos != 0);
+ protozero_assert(m_rollback_pos != size_is_known);
+ protozero_assert(m_data);
+ const auto length = pbf_length_type(buffer_customization<TBuffer>::size(m_data) - m_pos);
+
+ protozero_assert(buffer_customization<TBuffer>::size(m_data) >= m_pos - reserve_bytes);
+ const auto n = add_varint_to_buffer(buffer_customization<TBuffer>::at_pos(m_data, m_pos - reserve_bytes), length);
+
+ buffer_customization<TBuffer>::erase_range(m_data, m_pos - reserve_bytes + n, m_pos);
+ m_pos = 0;
+ }
+
+ void close_submessage() {
+ protozero_assert(m_data);
+ if (m_pos == 0 || m_rollback_pos == size_is_known) {
+ return;
+ }
+ if (buffer_customization<TBuffer>::size(m_data) - m_pos == 0) {
+ rollback_submessage();
+ } else {
+ commit_submessage();
+ }
+ }
+
+ void add_length_varint(pbf_tag_type tag, pbf_length_type length) {
+ add_field(tag, pbf_wire_type::length_delimited);
+ add_varint(length);
+ }
+
+public:
+
+ /**
+ * Create a writer using the specified buffer as a data store. The
+ * basic_pbf_writer stores a pointer to that buffer and adds all data to
+ * it. The buffer doesn't have to be empty. The basic_pbf_writer will just
+ * append data.
+ */
+ explicit basic_pbf_writer(TBuffer& buffer) noexcept :
+ m_data{&buffer} {
+ }
+
+ /**
+ * Create a writer without a data store. In this form the writer can not
+ * be used!
+ */
+ basic_pbf_writer() noexcept = default;
+
+ /**
+ * Construct a basic_pbf_writer for a submessage from the basic_pbf_writer
+ * of the parent message.
+ *
+ * @param parent_writer The basic_pbf_writer
+ * @param tag Tag (field number) of the field that will be written
+ * @param size Optional size of the submessage in bytes (use 0 for unknown).
+ * Setting this allows some optimizations but is only possible in
+ * a few very specific cases.
+ */
+ basic_pbf_writer(basic_pbf_writer& parent_writer, pbf_tag_type tag, std::size_t size = 0) :
+ m_data{parent_writer.m_data},
+ m_parent_writer{&parent_writer} {
+ m_parent_writer->open_submessage(tag, size);
+ }
+
+ /// A basic_pbf_writer object can not be copied
+ basic_pbf_writer(const basic_pbf_writer&) = delete;
+
+ /// A basic_pbf_writer object can not be copied
+ basic_pbf_writer& operator=(const basic_pbf_writer&) = delete;
+
+ /**
+ * A basic_pbf_writer object can be moved. After this the other
+ * basic_pbf_writer will be invalid.
+ */
+ basic_pbf_writer(basic_pbf_writer&& other) noexcept :
+ m_data{other.m_data},
+ m_parent_writer{other.m_parent_writer},
+ m_rollback_pos{other.m_rollback_pos},
+ m_pos{other.m_pos} {
+ other.m_data = nullptr;
+ other.m_parent_writer = nullptr;
+ other.m_rollback_pos = 0;
+ other.m_pos = 0;
+ }
+
+ /**
+ * A basic_pbf_writer object can be moved. After this the other
+ * basic_pbf_writer will be invalid.
+ */
+ basic_pbf_writer& operator=(basic_pbf_writer&& other) noexcept {
+ m_data = other.m_data;
+ m_parent_writer = other.m_parent_writer;
+ m_rollback_pos = other.m_rollback_pos;
+ m_pos = other.m_pos;
+ other.m_data = nullptr;
+ other.m_parent_writer = nullptr;
+ other.m_rollback_pos = 0;
+ other.m_pos = 0;
+ return *this;
+ }
+
+ ~basic_pbf_writer() noexcept {
+ try {
+ if (m_parent_writer != nullptr) {
+ m_parent_writer->close_submessage();
+ }
+ } catch (...) {
+ // This try/catch is used to make the destructor formally noexcept.
+ // close_submessage() is not noexcept, but will not throw the way
+ // it is called here, so we are good. But to be paranoid, call...
+ std::terminate();
+ }
+ }
+
+ /**
+ * Check if this writer is valid. A writer is invalid if it was default
+ * constructed, moved from, or if commit() has been called on it.
+ * Otherwise it is valid.
+ */
+ bool valid() const noexcept {
+ return m_data != nullptr;
+ }
+
+ /**
+ * Swap the contents of this object with the other.
+ *
+ * @param other Other object to swap data with.
+ */
+ void swap(basic_pbf_writer& other) noexcept {
+ using std::swap;
+ swap(m_data, other.m_data);
+ swap(m_parent_writer, other.m_parent_writer);
+ swap(m_rollback_pos, other.m_rollback_pos);
+ swap(m_pos, other.m_pos);
+ }
+
+ /**
+ * Reserve size bytes in the underlying message store in addition to
+ * whatever the message store already holds. So unlike
+ * the `std::string::reserve()` method this is not an absolute size,
+ * but additional memory that should be reserved.
+ *
+ * @param size Number of bytes to reserve in underlying message store.
+ */
+ void reserve(std::size_t size) {
+ protozero_assert(m_data);
+ buffer_customization<TBuffer>::reserve_additional(m_data, size);
+ }
+
+ /**
+ * Commit this submessage. This does the same as when the basic_pbf_writer
+ * goes out of scope and is destructed.
+ *
+ * @pre Must be a basic_pbf_writer of a submessage, ie one opened with the
+ * basic_pbf_writer constructor taking a parent message.
+ * @post The basic_pbf_writer is invalid and can't be used any more.
+ */
+ void commit() {
+ protozero_assert(m_parent_writer && "you can't call commit() on a basic_pbf_writer without a parent");
+ protozero_assert(m_pos == 0 && "you can't call commit() on a basic_pbf_writer that has an open nested submessage");
+ m_parent_writer->close_submessage();
+ m_parent_writer = nullptr;
+ m_data = nullptr;
+ }
+
+ /**
+ * Cancel writing of this submessage. The complete submessage will be
+ * removed as if it was never created and no fields were added.
+ *
+ * @pre Must be a basic_pbf_writer of a submessage, ie one opened with the
+ * basic_pbf_writer constructor taking a parent message.
+ * @post The basic_pbf_writer is invalid and can't be used any more.
+ */
+ void rollback() {
+ protozero_assert(m_parent_writer && "you can't call rollback() on a basic_pbf_writer without a parent");
+ protozero_assert(m_pos == 0 && "you can't call rollback() on a basic_pbf_writer that has an open nested submessage");
+ m_parent_writer->rollback_submessage();
+ m_parent_writer = nullptr;
+ m_data = nullptr;
+ }
+
+ ///@{
+ /**
+ * @name Scalar field writer functions
+ */
+
+ /**
+ * Add "bool" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_bool(pbf_tag_type tag, bool value) {
+ add_field(tag, pbf_wire_type::varint);
+ protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage");
+ protozero_assert(m_data);
+ m_data->push_back(static_cast<char>(value));
+ }
+
+ /**
+ * Add "enum" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_enum(pbf_tag_type tag, int32_t value) {
+ add_tagged_varint(tag, static_cast<uint64_t>(value));
+ }
+
+ /**
+ * Add "int32" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_int32(pbf_tag_type tag, int32_t value) {
+ add_tagged_varint(tag, static_cast<uint64_t>(value));
+ }
+
+ /**
+ * Add "sint32" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_sint32(pbf_tag_type tag, int32_t value) {
+ add_tagged_varint(tag, encode_zigzag32(value));
+ }
+
+ /**
+ * Add "uint32" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_uint32(pbf_tag_type tag, uint32_t value) {
+ add_tagged_varint(tag, value);
+ }
+
+ /**
+ * Add "int64" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_int64(pbf_tag_type tag, int64_t value) {
+ add_tagged_varint(tag, static_cast<uint64_t>(value));
+ }
+
+ /**
+ * Add "sint64" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_sint64(pbf_tag_type tag, int64_t value) {
+ add_tagged_varint(tag, encode_zigzag64(value));
+ }
+
+ /**
+ * Add "uint64" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_uint64(pbf_tag_type tag, uint64_t value) {
+ add_tagged_varint(tag, value);
+ }
+
+ /**
+ * Add "fixed32" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_fixed32(pbf_tag_type tag, uint32_t value) {
+ add_field(tag, pbf_wire_type::fixed32);
+ add_fixed<uint32_t>(value);
+ }
+
+ /**
+ * Add "sfixed32" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_sfixed32(pbf_tag_type tag, int32_t value) {
+ add_field(tag, pbf_wire_type::fixed32);
+ add_fixed<int32_t>(value);
+ }
+
+ /**
+ * Add "fixed64" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_fixed64(pbf_tag_type tag, uint64_t value) {
+ add_field(tag, pbf_wire_type::fixed64);
+ add_fixed<uint64_t>(value);
+ }
+
+ /**
+ * Add "sfixed64" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_sfixed64(pbf_tag_type tag, int64_t value) {
+ add_field(tag, pbf_wire_type::fixed64);
+ add_fixed<int64_t>(value);
+ }
+
+ /**
+ * Add "float" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_float(pbf_tag_type tag, float value) {
+ add_field(tag, pbf_wire_type::fixed32);
+ add_fixed<float>(value);
+ }
+
+ /**
+ * Add "double" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_double(pbf_tag_type tag, double value) {
+ add_field(tag, pbf_wire_type::fixed64);
+ add_fixed<double>(value);
+ }
+
+ /**
+ * Add "bytes" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Pointer to value to be written
+ * @param size Number of bytes to be written
+ */
+ void add_bytes(pbf_tag_type tag, const char* value, std::size_t size) {
+ protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage");
+ protozero_assert(m_data);
+ protozero_assert(size <= std::numeric_limits<pbf_length_type>::max());
+ add_length_varint(tag, static_cast<pbf_length_type>(size));
+ buffer_customization<TBuffer>::append(m_data, value, size);
+ }
+
+ /**
+ * Add "bytes" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_bytes(pbf_tag_type tag, const data_view& value) {
+ add_bytes(tag, value.data(), value.size());
+ }
+
+ /**
+ * Add "bytes" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_bytes(pbf_tag_type tag, const std::string& value) {
+ add_bytes(tag, value.data(), value.size());
+ }
+
+ /**
+ * Add "bytes" field to data. Bytes from the value are written until
+ * a null byte is encountered. The null byte is not added.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Pointer to zero-delimited value to be written
+ */
+ void add_bytes(pbf_tag_type tag, const char* value) {
+ add_bytes(tag, value, std::strlen(value));
+ }
+
+ /**
+ * Add "bytes" field to data using vectored input. All the data in the
+ * 2nd and further arguments is "concatenated" with only a single copy
+ * into the final buffer.
+ *
+ * This will work with objects of any type supporting the data() and
+ * size() methods like std::string or protozero::data_view.
+ *
+ * Example:
+ * @code
+ * std::string data1 = "abc";
+ * std::string data2 = "xyz";
+ * writer.add_bytes_vectored(1, data1, data2);
+ * @endcode
+ *
+ * @tparam Ts List of types supporting data() and size() methods.
+ * @param tag Tag (field number) of the field
+ * @param values List of objects of types Ts with data to be appended.
+ */
+ template <typename... Ts>
+ void add_bytes_vectored(pbf_tag_type tag, Ts&&... values) {
+ protozero_assert(m_pos == 0 && "you can't add fields to a parent basic_pbf_writer if there is an existing basic_pbf_writer for a submessage");
+ protozero_assert(m_data);
+ size_t sum_size = 0;
+ (void)std::initializer_list<size_t>{sum_size += values.size()...};
+ protozero_assert(sum_size <= std::numeric_limits<pbf_length_type>::max());
+ add_length_varint(tag, static_cast<pbf_length_type>(sum_size));
+ buffer_customization<TBuffer>::reserve_additional(m_data, sum_size);
+ (void)std::initializer_list<int>{(buffer_customization<TBuffer>::append(m_data, values.data(), values.size()), 0)...};
+ }
+
+ /**
+ * Add "string" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Pointer to value to be written
+ * @param size Number of bytes to be written
+ */
+ void add_string(pbf_tag_type tag, const char* value, std::size_t size) {
+ add_bytes(tag, value, size);
+ }
+
+ /**
+ * Add "string" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_string(pbf_tag_type tag, const data_view& value) {
+ add_bytes(tag, value.data(), value.size());
+ }
+
+ /**
+ * Add "string" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written
+ */
+ void add_string(pbf_tag_type tag, const std::string& value) {
+ add_bytes(tag, value.data(), value.size());
+ }
+
+ /**
+ * Add "string" field to data. Bytes from the value are written until
+ * a null byte is encountered. The null byte is not added.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Pointer to value to be written
+ */
+ void add_string(pbf_tag_type tag, const char* value) {
+ add_bytes(tag, value, std::strlen(value));
+ }
+
+ /**
+ * Add "message" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Pointer to message to be written
+ * @param size Length of the message
+ */
+ void add_message(pbf_tag_type tag, const char* value, std::size_t size) {
+ add_bytes(tag, value, size);
+ }
+
+ /**
+ * Add "message" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written. The value must be a complete message.
+ */
+ void add_message(pbf_tag_type tag, const data_view& value) {
+ add_bytes(tag, value.data(), value.size());
+ }
+
+ /**
+ * Add "message" field to data.
+ *
+ * @param tag Tag (field number) of the field
+ * @param value Value to be written. The value must be a complete message.
+ */
+ void add_message(pbf_tag_type tag, const std::string& value) {
+ add_bytes(tag, value.data(), value.size());
+ }
+
+ ///@}
+
+ ///@{
+ /**
+ * @name Repeated packed field writer functions
+ */
+
+ /**
+ * Add "repeated packed bool" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to bool.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_bool(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_varint(tag, first, last);
+ }
+
+ /**
+ * Add "repeated packed enum" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to int32_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_enum(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_varint(tag, first, last);
+ }
+
+ /**
+ * Add "repeated packed int32" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to int32_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_int32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_varint(tag, first, last);
+ }
+
+ /**
+ * Add "repeated packed sint32" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to int32_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_sint32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_svarint(tag, first, last);
+ }
+
+ /**
+ * Add "repeated packed uint32" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to uint32_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_uint32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_varint(tag, first, last);
+ }
+
+ /**
+ * Add "repeated packed int64" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to int64_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_int64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_varint(tag, first, last);
+ }
+
+ /**
+ * Add "repeated packed sint64" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to int64_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_sint64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_svarint(tag, first, last);
+ }
+
+ /**
+ * Add "repeated packed uint64" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to uint64_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_uint64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_varint(tag, first, last);
+ }
+
+ /**
+ * Add a "repeated packed" fixed-size field to data. The following
+ * fixed-size fields are available:
+ *
+ * uint32_t -> repeated packed fixed32
+ * int32_t -> repeated packed sfixed32
+ * uint64_t -> repeated packed fixed64
+ * int64_t -> repeated packed sfixed64
+ * double -> repeated packed double
+ * float -> repeated packed float
+ *
+ * @tparam ValueType One of the following types: (u)int32/64_t, double, float.
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename ValueType, typename InputIterator>
+ void add_packed_fixed(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ static_assert(std::is_same<ValueType, uint32_t>::value ||
+ std::is_same<ValueType, int32_t>::value ||
+ std::is_same<ValueType, int64_t>::value ||
+ std::is_same<ValueType, uint64_t>::value ||
+ std::is_same<ValueType, double>::value ||
+ std::is_same<ValueType, float>::value, "Only some types are allowed");
+ add_packed_fixed<ValueType, InputIterator>(tag, first, last,
+ typename std::iterator_traits<InputIterator>::iterator_category{});
+ }
+
+ /**
+ * Add "repeated packed fixed32" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to uint32_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_fixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_fixed<uint32_t, InputIterator>(tag, first, last,
+ typename std::iterator_traits<InputIterator>::iterator_category{});
+ }
+
+ /**
+ * Add "repeated packed sfixed32" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to int32_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_sfixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_fixed<int32_t, InputIterator>(tag, first, last,
+ typename std::iterator_traits<InputIterator>::iterator_category{});
+ }
+
+ /**
+ * Add "repeated packed fixed64" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to uint64_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_fixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_fixed<uint64_t, InputIterator>(tag, first, last,
+ typename std::iterator_traits<InputIterator>::iterator_category{});
+ }
+
+ /**
+ * Add "repeated packed sfixed64" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to int64_t.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_sfixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_fixed<int64_t, InputIterator>(tag, first, last,
+ typename std::iterator_traits<InputIterator>::iterator_category{});
+ }
+
+ /**
+ * Add "repeated packed float" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to float.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_float(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_fixed<float, InputIterator>(tag, first, last,
+ typename std::iterator_traits<InputIterator>::iterator_category{});
+ }
+
+ /**
+ * Add "repeated packed double" field to data.
+ *
+ * @tparam InputIterator A type satisfying the InputIterator concept.
+ * Dereferencing the iterator must yield a type assignable to double.
+ * @param tag Tag (field number) of the field
+ * @param first Iterator pointing to the beginning of the data
+ * @param last Iterator pointing one past the end of data
+ */
+ template <typename InputIterator>
+ void add_packed_double(pbf_tag_type tag, InputIterator first, InputIterator last) { // NOLINT(performance-unnecessary-value-param)
+ add_packed_fixed<double, InputIterator>(tag, first, last,
+ typename std::iterator_traits<InputIterator>::iterator_category{});
+ }
+
+ ///@}
+
+ template <typename B, typename T> friend class detail::packed_field_varint;
+ template <typename B, typename T> friend class detail::packed_field_svarint;
+ template <typename B, typename T> friend class detail::packed_field_fixed;
+
+}; // class basic_pbf_writer
+
+/**
+ * Swap two basic_pbf_writer objects.
+ *
+ * @param lhs First object.
+ * @param rhs Second object.
+ */
+template <typename TBuffer>
+inline void swap(basic_pbf_writer<TBuffer>& lhs, basic_pbf_writer<TBuffer>& rhs) noexcept {
+ lhs.swap(rhs);
+}
+
+namespace detail {
+
+ template <typename TBuffer>
+ class packed_field {
+
+ basic_pbf_writer<TBuffer> m_writer{};
+
+ public:
+
+ packed_field(const packed_field&) = delete;
+ packed_field& operator=(const packed_field&) = delete;
+
+ packed_field(packed_field&&) noexcept = default;
+ packed_field& operator=(packed_field&&) noexcept = default;
+
+ packed_field() = default;
+
+ packed_field(basic_pbf_writer<TBuffer>& parent_writer, pbf_tag_type tag) :
+ m_writer{parent_writer, tag} {
+ }
+
+ packed_field(basic_pbf_writer<TBuffer>& parent_writer, pbf_tag_type tag, std::size_t size) :
+ m_writer{parent_writer, tag, size} {
+ }
+
+ ~packed_field() noexcept = default;
+
+ bool valid() const noexcept {
+ return m_writer.valid();
+ }
+
+ void commit() {
+ m_writer.commit();
+ }
+
+ void rollback() {
+ m_writer.rollback();
+ }
+
+ basic_pbf_writer<TBuffer>& writer() noexcept {
+ return m_writer;
+ }
+
+ }; // class packed_field
+
+ template <typename TBuffer, typename T>
+ class packed_field_fixed : public packed_field<TBuffer> {
+
+ public:
+
+ packed_field_fixed() :
+ packed_field<TBuffer>{} {
+ }
+
+ template <typename P>
+ packed_field_fixed(basic_pbf_writer<TBuffer>& parent_writer, P tag) :
+ packed_field<TBuffer>{parent_writer, static_cast<pbf_tag_type>(tag)} {
+ }
+
+ template <typename P>
+ packed_field_fixed(basic_pbf_writer<TBuffer>& parent_writer, P tag, std::size_t size) :
+ packed_field<TBuffer>{parent_writer, static_cast<pbf_tag_type>(tag), size * sizeof(T)} {
+ }
+
+ void add_element(T value) {
+ this->writer().template add_fixed<T>(value);
+ }
+
+ }; // class packed_field_fixed
+
+ template <typename TBuffer, typename T>
+ class packed_field_varint : public packed_field<TBuffer> {
+
+ public:
+
+ packed_field_varint() :
+ packed_field<TBuffer>{} {
+ }
+
+ template <typename P>
+ packed_field_varint(basic_pbf_writer<TBuffer>& parent_writer, P tag) :
+ packed_field<TBuffer>{parent_writer, static_cast<pbf_tag_type>(tag)} {
+ }
+
+ void add_element(T value) {
+ this->writer().add_varint(uint64_t(value));
+ }
+
+ }; // class packed_field_varint
+
+ template <typename TBuffer, typename T>
+ class packed_field_svarint : public packed_field<TBuffer> {
+
+ public:
+
+ packed_field_svarint() :
+ packed_field<TBuffer>{} {
+ }
+
+ template <typename P>
+ packed_field_svarint(basic_pbf_writer<TBuffer>& parent_writer, P tag) :
+ packed_field<TBuffer>{parent_writer, static_cast<pbf_tag_type>(tag)} {
+ }
+
+ void add_element(T value) {
+ this->writer().add_varint(encode_zigzag64(value));
+ }
+
+ }; // class packed_field_svarint
+
+} // end namespace detail
+
+} // end namespace protozero
+
+#endif // PROTOZERO_BASIC_PBF_WRITER_HPP
diff --git a/thirdparty/protozero/include/protozero/buffer_fixed.hpp b/thirdparty/protozero/include/protozero/buffer_fixed.hpp
new file mode 100644
index 000000000..b2e6d1d27
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/buffer_fixed.hpp
@@ -0,0 +1,222 @@
+#ifndef PROTOZERO_BUFFER_FIXED_HPP
+#define PROTOZERO_BUFFER_FIXED_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file buffer_fixed.hpp
+ *
+ * @brief Contains the fixed_size_buffer_adaptor class.
+ */
+
+#include "buffer_tmpl.hpp"
+#include "config.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <iterator>
+#include <stdexcept>
+
+namespace protozero {
+
+/**
+ * This class can be used instead of std::string if you want to create a
+ * vector tile in a fixed-size buffer. Any operation that needs more space
+ * than is available will fail with a std::length_error exception.
+ */
+class fixed_size_buffer_adaptor {
+
+ char* m_data;
+ std::size_t m_capacity;
+ std::size_t m_size = 0;
+
+public:
+
+ /// @cond usual container typedefs not documented
+
+ using size_type = std::size_t;
+
+ using value_type = char;
+ using reference = value_type&;
+ using const_reference = const value_type&;
+ using pointer = value_type*;
+ using const_pointer = const value_type*;
+
+ using iterator = pointer;
+ using const_iterator = const_pointer;
+
+ /// @endcond
+
+ /**
+ * Constructor.
+ *
+ * @param data Pointer to some memory allocated for the buffer.
+ * @param capacity Number of bytes available.
+ */
+ fixed_size_buffer_adaptor(char* data, std::size_t capacity) noexcept :
+ m_data(data),
+ m_capacity(capacity) {
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param container Some container class supporting the member functions
+ * data() and size().
+ */
+ template <typename T>
+ explicit fixed_size_buffer_adaptor(T& container) :
+ m_data(container.data()),
+ m_capacity(container.size()) {
+ }
+
+ /// Returns a pointer to the data in the buffer.
+ const char* data() const noexcept {
+ return m_data;
+ }
+
+ /// Returns a pointer to the data in the buffer.
+ char* data() noexcept {
+ return m_data;
+ }
+
+ /// The capacity this buffer was created with.
+ std::size_t capacity() const noexcept {
+ return m_capacity;
+ }
+
+ /// The number of bytes used in the buffer. Always <= capacity().
+ std::size_t size() const noexcept {
+ return m_size;
+ }
+
+ /// Return iterator to beginning of data.
+ char* begin() noexcept {
+ return m_data;
+ }
+
+ /// Return iterator to beginning of data.
+ const char* begin() const noexcept {
+ return m_data;
+ }
+
+ /// Return iterator to beginning of data.
+ const char* cbegin() const noexcept {
+ return m_data;
+ }
+
+ /// Return iterator to end of data.
+ char* end() noexcept {
+ return m_data + m_size;
+ }
+
+ /// Return iterator to end of data.
+ const char* end() const noexcept {
+ return m_data + m_size;
+ }
+
+ /// Return iterator to end of data.
+ const char* cend() const noexcept {
+ return m_data + m_size;
+ }
+
+/// @cond INTERNAL
+
+ // Do not rely on anything beyond this point
+
+ void append(const char* data, std::size_t count) {
+ if (m_size + count > m_capacity) {
+ throw std::length_error{"fixed size data store exhausted"};
+ }
+ std::copy_n(data, count, m_data + m_size);
+ m_size += count;
+ }
+
+ void append_zeros(std::size_t count) {
+ if (m_size + count > m_capacity) {
+ throw std::length_error{"fixed size data store exhausted"};
+ }
+ std::fill_n(m_data + m_size, count, '\0');
+ m_size += count;
+ }
+
+ void resize(std::size_t size) {
+ protozero_assert(size < m_size);
+ if (size > m_capacity) {
+ throw std::length_error{"fixed size data store exhausted"};
+ }
+ m_size = size;
+ }
+
+ void erase_range(std::size_t from, std::size_t to) {
+ protozero_assert(from <= m_size);
+ protozero_assert(to <= m_size);
+ protozero_assert(from < to);
+ std::copy(m_data + to, m_data + m_size, m_data + from);
+ m_size -= (to - from);
+ }
+
+ char* at_pos(std::size_t pos) {
+ protozero_assert(pos <= m_size);
+ return m_data + pos;
+ }
+
+ void push_back(char ch) {
+ if (m_size >= m_capacity) {
+ throw std::length_error{"fixed size data store exhausted"};
+ }
+ m_data[m_size++] = ch;
+ }
+/// @endcond
+
+}; // class fixed_size_buffer_adaptor
+
+/// @cond INTERNAL
+template <>
+struct buffer_customization<fixed_size_buffer_adaptor> {
+
+ static std::size_t size(const fixed_size_buffer_adaptor* buffer) noexcept {
+ return buffer->size();
+ }
+
+ static void append(fixed_size_buffer_adaptor* buffer, const char* data, std::size_t count) {
+ buffer->append(data, count);
+ }
+
+ static void append_zeros(fixed_size_buffer_adaptor* buffer, std::size_t count) {
+ buffer->append_zeros(count);
+ }
+
+ static void resize(fixed_size_buffer_adaptor* buffer, std::size_t size) {
+ buffer->resize(size);
+ }
+
+ static void reserve_additional(fixed_size_buffer_adaptor* /*buffer*/, std::size_t /*size*/) {
+ /* nothing to be done for fixed-size buffers */
+ }
+
+ static void erase_range(fixed_size_buffer_adaptor* buffer, std::size_t from, std::size_t to) {
+ buffer->erase_range(from, to);
+ }
+
+ static char* at_pos(fixed_size_buffer_adaptor* buffer, std::size_t pos) {
+ return buffer->at_pos(pos);
+ }
+
+ static void push_back(fixed_size_buffer_adaptor* buffer, char ch) {
+ buffer->push_back(ch);
+ }
+
+};
+/// @endcond
+
+} // namespace protozero
+
+#endif // PROTOZERO_BUFFER_FIXED_HPP
diff --git a/thirdparty/protozero/include/protozero/buffer_string.hpp b/thirdparty/protozero/include/protozero/buffer_string.hpp
new file mode 100644
index 000000000..c620879e4
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/buffer_string.hpp
@@ -0,0 +1,78 @@
+#ifndef PROTOZERO_BUFFER_STRING_HPP
+#define PROTOZERO_BUFFER_STRING_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file buffer_string.hpp
+ *
+ * @brief Contains the customization points for buffer implementation based
+ * on std::string
+ */
+
+#include "buffer_tmpl.hpp"
+#include "config.hpp"
+
+#include <cstddef>
+#include <iterator>
+#include <string>
+
+namespace protozero {
+
+// Implementation of buffer customizations points for std::string
+
+/// @cond INTERNAL
+template <>
+struct buffer_customization<std::string> {
+
+ static std::size_t size(const std::string* buffer) noexcept {
+ return buffer->size();
+ }
+
+ static void append(std::string* buffer, const char* data, std::size_t count) {
+ buffer->append(data, count);
+ }
+
+ static void append_zeros(std::string* buffer, std::size_t count) {
+ buffer->append(count, '\0');
+ }
+
+ static void resize(std::string* buffer, std::size_t size) {
+ protozero_assert(size < buffer->size());
+ buffer->resize(size);
+ }
+
+ static void reserve_additional(std::string* buffer, std::size_t size) {
+ buffer->reserve(buffer->size() + size);
+ }
+
+ static void erase_range(std::string* buffer, std::size_t from, std::size_t to) {
+ protozero_assert(from <= buffer->size());
+ protozero_assert(to <= buffer->size());
+ protozero_assert(from <= to);
+ buffer->erase(buffer->begin() + static_cast<std::string::difference_type>(from),
+ buffer->begin() + static_cast<std::string::difference_type>(to));
+ }
+
+ static char* at_pos(std::string* buffer, std::size_t pos) {
+ protozero_assert(pos <= buffer->size());
+ return (&*buffer->begin()) + pos;
+ }
+
+ static void push_back(std::string* buffer, char ch) {
+ buffer->push_back(ch);
+ }
+
+};
+/// @endcond
+
+} // namespace protozero
+
+#endif // PROTOZERO_BUFFER_STRING_HPP
diff --git a/thirdparty/protozero/include/protozero/buffer_tmpl.hpp b/thirdparty/protozero/include/protozero/buffer_tmpl.hpp
new file mode 100644
index 000000000..ac223996d
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/buffer_tmpl.hpp
@@ -0,0 +1,113 @@
+#ifndef PROTOZERO_BUFFER_TMPL_HPP
+#define PROTOZERO_BUFFER_TMPL_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file buffer_tmpl.hpp
+ *
+ * @brief Contains the customization points for buffer implementations.
+ */
+
+#include <cstddef>
+#include <iterator>
+#include <string>
+
+namespace protozero {
+
+// Implementation of buffer customizations points for std::string
+
+/// @cond INTERNAL
+template <typename T>
+struct buffer_customization {
+
+ /**
+ * Get the number of bytes currently used in the buffer.
+ *
+ * @param buffer Pointer to the buffer.
+ * @returns number of bytes used in the buffer.
+ */
+ static std::size_t size(const std::string* buffer);
+
+ /**
+ * Append count bytes from data to the buffer.
+ *
+ * @param buffer Pointer to the buffer.
+ * @param data Pointer to the data.
+ * @param count Number of bytes to be added to the buffer.
+ */
+ static void append(std::string* buffer, const char* data, std::size_t count);
+
+ /**
+ * Append count zero bytes to the buffer.
+ *
+ * @param buffer Pointer to the buffer.
+ * @param count Number of bytes to be added to the buffer.
+ */
+ static void append_zeros(std::string* buffer, std::size_t count);
+
+ /**
+ * Shrink the buffer to the specified size. The new size will always be
+ * smaller than the current size.
+ *
+ * @param buffer Pointer to the buffer.
+ * @param size New size of the buffer.
+ *
+ * @pre size < current size of buffer
+ */
+ static void resize(std::string* buffer, std::size_t size);
+
+ /**
+ * Reserve an additional size bytes for use in the buffer. This is used for
+ * variable-sized buffers to tell the buffer implementation that soon more
+ * memory will be used. The implementation can ignore this.
+ *
+ * @param buffer Pointer to the buffer.
+ * @param size Number of bytes to reserve.
+ */
+ static void reserve_additional(std::string* buffer, std::size_t size);
+
+ /**
+ * Delete data from the buffer. This must move back the data after the
+ * part being deleted and resize the buffer accordingly.
+ *
+ * @param buffer Pointer to the buffer.
+ * @param from Offset into the buffer where we want to erase from.
+ * @param to Offset into the buffer one past the last byte we want to erase.
+ *
+ * @pre from, to <= size of the buffer, from < to
+ */
+ static void erase_range(std::string* buffer, std::size_t from, std::size_t to);
+
+ /**
+ * Return a pointer to the memory at the specified position in the buffer.
+ *
+ * @param buffer Pointer to the buffer.
+ * @param pos The position in the buffer.
+ * @returns pointer to the memory in the buffer at the specified position.
+ *
+ * @pre pos <= size of the buffer
+ */
+ static char* at_pos(std::string* buffer, std::size_t pos);
+
+ /**
+ * Add a char to the buffer incrementing the number of chars in the buffer.
+ *
+ * @param buffer Pointer to the buffer.
+ * @param ch The character to add.
+ */
+ static void push_back(std::string* buffer, char ch);
+
+};
+/// @endcond
+
+} // namespace protozero
+
+#endif // PROTOZERO_BUFFER_TMPL_HPP
diff --git a/thirdparty/protozero/include/protozero/buffer_vector.hpp b/thirdparty/protozero/include/protozero/buffer_vector.hpp
new file mode 100644
index 000000000..c163300c5
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/buffer_vector.hpp
@@ -0,0 +1,78 @@
+#ifndef PROTOZERO_BUFFER_VECTOR_HPP
+#define PROTOZERO_BUFFER_VECTOR_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file buffer_vector.hpp
+ *
+ * @brief Contains the customization points for buffer implementation based
+ * on std::vector<char>
+ */
+
+#include "buffer_tmpl.hpp"
+#include "config.hpp"
+
+#include <cstddef>
+#include <iterator>
+#include <vector>
+
+namespace protozero {
+
+// Implementation of buffer customizations points for std::vector<char>
+
+/// @cond INTERNAL
+template <>
+struct buffer_customization<std::vector<char>> {
+
+ static std::size_t size(const std::vector<char>* buffer) noexcept {
+ return buffer->size();
+ }
+
+ static void append(std::vector<char>* buffer, const char* data, std::size_t count) {
+ buffer->insert(buffer->end(), data, data + count);
+ }
+
+ static void append_zeros(std::vector<char>* buffer, std::size_t count) {
+ buffer->insert(buffer->end(), count, '\0');
+ }
+
+ static void resize(std::vector<char>* buffer, std::size_t size) {
+ protozero_assert(size < buffer->size());
+ buffer->resize(size);
+ }
+
+ static void reserve_additional(std::vector<char>* buffer, std::size_t size) {
+ buffer->reserve(buffer->size() + size);
+ }
+
+ static void erase_range(std::vector<char>* buffer, std::size_t from, std::size_t to) {
+ protozero_assert(from <= buffer->size());
+ protozero_assert(to <= buffer->size());
+ protozero_assert(from <= to);
+ buffer->erase(std::next(buffer->begin(), static_cast<std::string::iterator::difference_type>(from)),
+ std::next(buffer->begin(), static_cast<std::string::iterator::difference_type>(to)));
+ }
+
+ static char* at_pos(std::vector<char>* buffer, std::size_t pos) {
+ protozero_assert(pos <= buffer->size());
+ return (&*buffer->begin()) + pos;
+ }
+
+ static void push_back(std::vector<char>* buffer, char ch) {
+ buffer->push_back(ch);
+ }
+
+};
+/// @endcond
+
+} // namespace protozero
+
+#endif // PROTOZERO_BUFFER_VECTOR_HPP
diff --git a/thirdparty/protozero/include/protozero/byteswap.hpp b/thirdparty/protozero/include/protozero/byteswap.hpp
new file mode 100644
index 000000000..75cae6910
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/byteswap.hpp
@@ -0,0 +1,108 @@
+#ifndef PROTOZERO_BYTESWAP_HPP
+#define PROTOZERO_BYTESWAP_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file byteswap.hpp
+ *
+ * @brief Contains functions to swap bytes in values (for different endianness).
+ */
+
+#include "config.hpp"
+
+#include <cstdint>
+#include <cstring>
+
+namespace protozero {
+namespace detail {
+
+inline uint32_t byteswap_impl(uint32_t value) noexcept {
+#ifdef PROTOZERO_USE_BUILTIN_BSWAP
+ return __builtin_bswap32(value);
+#else
+ return ((value & 0xff000000U) >> 24U) |
+ ((value & 0x00ff0000U) >> 8U) |
+ ((value & 0x0000ff00U) << 8U) |
+ ((value & 0x000000ffU) << 24U);
+#endif
+}
+
+inline uint64_t byteswap_impl(uint64_t value) noexcept {
+#ifdef PROTOZERO_USE_BUILTIN_BSWAP
+ return __builtin_bswap64(value);
+#else
+ return ((value & 0xff00000000000000ULL) >> 56U) |
+ ((value & 0x00ff000000000000ULL) >> 40U) |
+ ((value & 0x0000ff0000000000ULL) >> 24U) |
+ ((value & 0x000000ff00000000ULL) >> 8U) |
+ ((value & 0x00000000ff000000ULL) << 8U) |
+ ((value & 0x0000000000ff0000ULL) << 24U) |
+ ((value & 0x000000000000ff00ULL) << 40U) |
+ ((value & 0x00000000000000ffULL) << 56U);
+#endif
+}
+
+} // end namespace detail
+
+/// byteswap the data pointed to by ptr in-place.
+inline void byteswap_inplace(uint32_t* ptr) noexcept {
+ *ptr = detail::byteswap_impl(*ptr);
+}
+
+/// byteswap the data pointed to by ptr in-place.
+inline void byteswap_inplace(uint64_t* ptr) noexcept {
+ *ptr = detail::byteswap_impl(*ptr);
+}
+
+/// byteswap the data pointed to by ptr in-place.
+inline void byteswap_inplace(int32_t* ptr) noexcept {
+ auto* bptr = reinterpret_cast<uint32_t*>(ptr);
+ *bptr = detail::byteswap_impl(*bptr);
+}
+
+/// byteswap the data pointed to by ptr in-place.
+inline void byteswap_inplace(int64_t* ptr) noexcept {
+ auto* bptr = reinterpret_cast<uint64_t*>(ptr);
+ *bptr = detail::byteswap_impl(*bptr);
+}
+
+/// byteswap the data pointed to by ptr in-place.
+inline void byteswap_inplace(float* ptr) noexcept {
+ static_assert(sizeof(float) == 4, "Expecting four byte float");
+
+ uint32_t tmp = 0;
+ std::memcpy(&tmp, ptr, 4);
+ tmp = detail::byteswap_impl(tmp); // uint32 overload
+ std::memcpy(ptr, &tmp, 4);
+}
+
+/// byteswap the data pointed to by ptr in-place.
+inline void byteswap_inplace(double* ptr) noexcept {
+ static_assert(sizeof(double) == 8, "Expecting eight byte double");
+
+ uint64_t tmp = 0;
+ std::memcpy(&tmp, ptr, 8);
+ tmp = detail::byteswap_impl(tmp); // uint64 overload
+ std::memcpy(ptr, &tmp, 8);
+}
+
+namespace detail {
+
+ // Added for backwards compatibility with any code that might use this
+ // function (even if it shouldn't have). Will be removed in a later
+ // version of protozero.
+ using ::protozero::byteswap_inplace;
+
+} // end namespace detail
+
+} // end namespace protozero
+
+#endif // PROTOZERO_BYTESWAP_HPP
diff --git a/thirdparty/protozero/include/protozero/config.hpp b/thirdparty/protozero/include/protozero/config.hpp
new file mode 100644
index 000000000..6fc774904
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/config.hpp
@@ -0,0 +1,48 @@
+#ifndef PROTOZERO_CONFIG_HPP
+#define PROTOZERO_CONFIG_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+#include <cassert>
+
+/**
+ * @file config.hpp
+ *
+ * @brief Contains macro checks for different configurations.
+ */
+
+#define PROTOZERO_LITTLE_ENDIAN 1234
+#define PROTOZERO_BIG_ENDIAN 4321
+
+// Find out which byte order the machine has.
+#if defined(__BYTE_ORDER)
+# if (__BYTE_ORDER == __LITTLE_ENDIAN)
+# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN
+# endif
+# if (__BYTE_ORDER == __BIG_ENDIAN)
+# define PROTOZERO_BYTE_ORDER PROTOZERO_BIG_ENDIAN
+# endif
+#else
+// This probably isn't a very good default, but might do until we figure
+// out something better.
+# define PROTOZERO_BYTE_ORDER PROTOZERO_LITTLE_ENDIAN
+#endif
+
+// Check whether __builtin_bswap is available
+#if defined(__GNUC__) || defined(__clang__)
+# define PROTOZERO_USE_BUILTIN_BSWAP
+#endif
+
+// Wrapper for assert() used for testing
+#ifndef protozero_assert
+# define protozero_assert(x) assert(x)
+#endif
+
+#endif // PROTOZERO_CONFIG_HPP
diff --git a/thirdparty/protozero/include/protozero/data_view.hpp b/thirdparty/protozero/include/protozero/data_view.hpp
new file mode 100644
index 000000000..815ceb3d2
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/data_view.hpp
@@ -0,0 +1,236 @@
+#ifndef PROTOZERO_DATA_VIEW_HPP
+#define PROTOZERO_DATA_VIEW_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file data_view.hpp
+ *
+ * @brief Contains the implementation of the data_view class.
+ */
+
+#include "config.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+#include <string>
+#include <utility>
+
+namespace protozero {
+
+#ifdef PROTOZERO_USE_VIEW
+using data_view = PROTOZERO_USE_VIEW;
+#else
+
+/**
+ * Holds a pointer to some data and a length.
+ *
+ * This class is supposed to be compatible with the std::string_view
+ * that will be available in C++17.
+ */
+class data_view {
+
+ const char* m_data = nullptr;
+ std::size_t m_size = 0;
+
+public:
+
+ /**
+ * Default constructor. Construct an empty data_view.
+ */
+ constexpr data_view() noexcept = default;
+
+ /**
+ * Create data_view from pointer and size.
+ *
+ * @param ptr Pointer to the data.
+ * @param length Length of the data.
+ */
+ constexpr data_view(const char* ptr, std::size_t length) noexcept
+ : m_data{ptr},
+ m_size{length} {
+ }
+
+ /**
+ * Create data_view from string.
+ *
+ * @param str String with the data.
+ */
+ data_view(const std::string& str) noexcept // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
+ : m_data{str.data()},
+ m_size{str.size()} {
+ }
+
+ /**
+ * Create data_view from zero-terminated string.
+ *
+ * @param ptr Pointer to the data.
+ */
+ data_view(const char* ptr) noexcept // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
+ : m_data{ptr},
+ m_size{std::strlen(ptr)} {
+ }
+
+ /**
+ * Swap the contents of this object with the other.
+ *
+ * @param other Other object to swap data with.
+ */
+ void swap(data_view& other) noexcept {
+ using std::swap;
+ swap(m_data, other.m_data);
+ swap(m_size, other.m_size);
+ }
+
+ /// Return pointer to data.
+ constexpr const char* data() const noexcept {
+ return m_data;
+ }
+
+ /// Return length of data in bytes.
+ constexpr std::size_t size() const noexcept {
+ return m_size;
+ }
+
+ /// Returns true if size is 0.
+ constexpr bool empty() const noexcept {
+ return m_size == 0;
+ }
+
+#ifndef PROTOZERO_STRICT_API
+ /**
+ * Convert data view to string.
+ *
+ * @pre Must not be default constructed data_view.
+ *
+ * @deprecated to_string() is not available in C++17 string_view so it
+ * should not be used to make conversion to that class easier
+ * in the future.
+ */
+ std::string to_string() const {
+ protozero_assert(m_data);
+ return {m_data, m_size};
+ }
+#endif
+
+ /**
+ * Convert data view to string.
+ *
+ * @pre Must not be default constructed data_view.
+ */
+ explicit operator std::string() const {
+ protozero_assert(m_data);
+ return {m_data, m_size};
+ }
+
+ /**
+ * Compares the contents of this object with the given other object.
+ *
+ * @returns 0 if they are the same, <0 if this object is smaller than
+ * the other or >0 if it is larger. If both objects have the
+ * same size returns <0 if this object is lexicographically
+ * before the other, >0 otherwise.
+ *
+ * @pre Must not be default constructed data_view.
+ */
+ int compare(data_view other) const noexcept {
+ assert(m_data && other.m_data);
+ const int cmp = std::memcmp(data(), other.data(),
+ std::min(size(), other.size()));
+ if (cmp == 0) {
+ if (size() == other.size()) {
+ return 0;
+ }
+ return size() < other.size() ? -1 : 1;
+ }
+ return cmp;
+ }
+
+}; // class data_view
+
+/**
+ * Swap two data_view objects.
+ *
+ * @param lhs First object.
+ * @param rhs Second object.
+ */
+inline void swap(data_view& lhs, data_view& rhs) noexcept {
+ lhs.swap(rhs);
+}
+
+/**
+ * Two data_view instances are equal if they have the same size and the
+ * same content.
+ *
+ * @param lhs First object.
+ * @param rhs Second object.
+ */
+constexpr bool operator==(const data_view lhs, const data_view rhs) noexcept {
+ return lhs.size() == rhs.size() &&
+ std::equal(lhs.data(), lhs.data() + lhs.size(), rhs.data());
+}
+
+/**
+ * Two data_view instances are not equal if they have different sizes or the
+ * content differs.
+ *
+ * @param lhs First object.
+ * @param rhs Second object.
+ */
+constexpr bool operator!=(const data_view lhs, const data_view rhs) noexcept {
+ return !(lhs == rhs);
+}
+
+/**
+ * Returns true if lhs.compare(rhs) < 0.
+ *
+ * @param lhs First object.
+ * @param rhs Second object.
+ */
+inline bool operator<(const data_view lhs, const data_view rhs) noexcept {
+ return lhs.compare(rhs) < 0;
+}
+
+/**
+ * Returns true if lhs.compare(rhs) <= 0.
+ *
+ * @param lhs First object.
+ * @param rhs Second object.
+ */
+inline bool operator<=(const data_view lhs, const data_view rhs) noexcept {
+ return lhs.compare(rhs) <= 0;
+}
+
+/**
+ * Returns true if lhs.compare(rhs) > 0.
+ *
+ * @param lhs First object.
+ * @param rhs Second object.
+ */
+inline bool operator>(const data_view lhs, const data_view rhs) noexcept {
+ return lhs.compare(rhs) > 0;
+}
+
+/**
+ * Returns true if lhs.compare(rhs) >= 0.
+ *
+ * @param lhs First object.
+ * @param rhs Second object.
+ */
+inline bool operator>=(const data_view lhs, const data_view rhs) noexcept {
+ return lhs.compare(rhs) >= 0;
+}
+
+#endif
+
+} // end namespace protozero
+
+#endif // PROTOZERO_DATA_VIEW_HPP
diff --git a/thirdparty/protozero/include/protozero/exception.hpp b/thirdparty/protozero/include/protozero/exception.hpp
new file mode 100644
index 000000000..a3cd0f15d
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/exception.hpp
@@ -0,0 +1,101 @@
+#ifndef PROTOZERO_EXCEPTION_HPP
+#define PROTOZERO_EXCEPTION_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file exception.hpp
+ *
+ * @brief Contains the exceptions used in the protozero library.
+ */
+
+#include <exception>
+
+/**
+ * @brief All parts of the protozero header-only library are in this namespace.
+ */
+namespace protozero {
+
+/**
+ * All exceptions explicitly thrown by the functions of the protozero library
+ * derive from this exception.
+ */
+struct exception : std::exception {
+ /// Returns the explanatory string.
+ const char* what() const noexcept override {
+ return "pbf exception";
+ }
+};
+
+/**
+ * This exception is thrown when parsing a varint thats larger than allowed.
+ * This should never happen unless the data is corrupted.
+ */
+struct varint_too_long_exception : exception {
+ /// Returns the explanatory string.
+ const char* what() const noexcept override {
+ return "varint too long exception";
+ }
+};
+
+/**
+ * This exception is thrown when the wire type of a pdf field is unknown.
+ * This should never happen unless the data is corrupted.
+ */
+struct unknown_pbf_wire_type_exception : exception {
+ /// Returns the explanatory string.
+ const char* what() const noexcept override {
+ return "unknown pbf field type exception";
+ }
+};
+
+/**
+ * This exception is thrown when we are trying to read a field and there
+ * are not enough bytes left in the buffer to read it. Almost all functions
+ * of the pbf_reader class can throw this exception.
+ *
+ * This should never happen unless the data is corrupted or you have
+ * initialized the pbf_reader object with incomplete data.
+ */
+struct end_of_buffer_exception : exception {
+ /// Returns the explanatory string.
+ const char* what() const noexcept override {
+ return "end of buffer exception";
+ }
+};
+
+/**
+ * This exception is thrown when a tag has an invalid value. Tags must be
+ * unsigned integers between 1 and 2^29-1. Tags between 19000 and 19999 are
+ * not allowed. See
+ * https://developers.google.com/protocol-buffers/docs/proto#assigning-tags
+ */
+struct invalid_tag_exception : exception {
+ /// Returns the explanatory string.
+ const char* what() const noexcept override {
+ return "invalid tag exception";
+ }
+};
+
+/**
+ * This exception is thrown when a length field of a packed repeated field is
+ * invalid. For fixed size types the length must be a multiple of the size of
+ * the type.
+ */
+struct invalid_length_exception : exception {
+ /// Returns the explanatory string.
+ const char* what() const noexcept override {
+ return "invalid length exception";
+ }
+};
+
+} // end namespace protozero
+
+#endif // PROTOZERO_EXCEPTION_HPP
diff --git a/thirdparty/protozero/include/protozero/iterators.hpp b/thirdparty/protozero/include/protozero/iterators.hpp
new file mode 100644
index 000000000..ee8ef8ecf
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/iterators.hpp
@@ -0,0 +1,481 @@
+#ifndef PROTOZERO_ITERATORS_HPP
+#define PROTOZERO_ITERATORS_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file iterators.hpp
+ *
+ * @brief Contains the iterators for access to packed repeated fields.
+ */
+
+#include "config.hpp"
+#include "varint.hpp"
+
+#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
+# include <protozero/byteswap.hpp>
+#endif
+
+#include <algorithm>
+#include <cstring>
+#include <iterator>
+#include <utility>
+
+namespace protozero {
+
+/**
+ * A range of iterators based on std::pair. Created from beginning and
+ * end iterators. Used as a return type from some pbf_reader methods
+ * that is easy to use with range-based for loops.
+ */
+template <typename T, typename P = std::pair<T, T>>
+class iterator_range :
+#ifdef PROTOZERO_STRICT_API
+ protected
+#else
+ public
+#endif
+ P {
+
+public:
+
+ /// The type of the iterators in this range.
+ using iterator = T;
+
+ /// The value type of the underlying iterator.
+ using value_type = typename std::iterator_traits<T>::value_type;
+
+ /**
+ * Default constructor. Create empty iterator_range.
+ */
+ constexpr iterator_range() :
+ P{iterator{}, iterator{}} {
+ }
+
+ /**
+ * Create iterator range from two iterators.
+ *
+ * @param first_iterator Iterator to beginning of range.
+ * @param last_iterator Iterator to end of range.
+ */
+ constexpr iterator_range(iterator&& first_iterator, iterator&& last_iterator) :
+ P{std::forward<iterator>(first_iterator),
+ std::forward<iterator>(last_iterator)} {
+ }
+
+ /// Return iterator to beginning of range.
+ constexpr iterator begin() const noexcept {
+ return this->first;
+ }
+
+ /// Return iterator to end of range.
+ constexpr iterator end() const noexcept {
+ return this->second;
+ }
+
+ /// Return iterator to beginning of range.
+ constexpr iterator cbegin() const noexcept {
+ return this->first;
+ }
+
+ /// Return iterator to end of range.
+ constexpr iterator cend() const noexcept {
+ return this->second;
+ }
+
+ /**
+ * Return true if this range is empty.
+ *
+ * Complexity: Constant.
+ */
+ constexpr bool empty() const noexcept {
+ return begin() == end();
+ }
+
+ /**
+ * Get the size of the range, ie the number of elements it contains.
+ *
+ * Complexity: Constant or linear depending on the underlaying iterator.
+ */
+ std::size_t size() const noexcept {
+ return static_cast<size_t>(std::distance(begin(), end()));
+ }
+
+ /**
+ * Get element at the beginning of the range.
+ *
+ * @pre Range must not be empty.
+ */
+ value_type front() const {
+ protozero_assert(!empty());
+ return *(this->first);
+ }
+
+ /**
+ * Advance beginning of range by one.
+ *
+ * @pre Range must not be empty.
+ */
+ void drop_front() {
+ protozero_assert(!empty());
+ ++this->first;
+ }
+
+ /**
+ * Swap the contents of this range with the other.
+ *
+ * @param other Other range to swap data with.
+ */
+ void swap(iterator_range& other) noexcept {
+ using std::swap;
+ swap(this->first, other.first);
+ swap(this->second, other.second);
+ }
+
+}; // struct iterator_range
+
+/**
+ * Swap two iterator_ranges.
+ *
+ * @param lhs First range.
+ * @param rhs Second range.
+ */
+template <typename T>
+inline void swap(iterator_range<T>& lhs, iterator_range<T>& rhs) noexcept {
+ lhs.swap(rhs);
+}
+
+/**
+ * A forward iterator used for accessing packed repeated fields of fixed
+ * length (fixed32, sfixed32, float, double).
+ */
+template <typename T>
+class const_fixed_iterator {
+
+ /// Pointer to current iterator position
+ const char* m_data = nullptr;
+
+public:
+
+ /// @cond usual iterator functions not documented
+
+ using iterator_category = std::random_access_iterator_tag;
+ using value_type = T;
+ using difference_type = std::ptrdiff_t;
+ using pointer = value_type*;
+ using reference = value_type&;
+
+ const_fixed_iterator() noexcept = default;
+
+ explicit const_fixed_iterator(const char* data) noexcept :
+ m_data{data} {
+ }
+
+ const_fixed_iterator(const const_fixed_iterator&) noexcept = default;
+ const_fixed_iterator(const_fixed_iterator&&) noexcept = default;
+
+ const_fixed_iterator& operator=(const const_fixed_iterator&) noexcept = default;
+ const_fixed_iterator& operator=(const_fixed_iterator&&) noexcept = default;
+
+ ~const_fixed_iterator() noexcept = default;
+
+ value_type operator*() const noexcept {
+ value_type result;
+ std::memcpy(&result, m_data, sizeof(value_type));
+#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
+ byteswap_inplace(&result);
+#endif
+ return result;
+ }
+
+ const_fixed_iterator& operator++() noexcept {
+ m_data += sizeof(value_type);
+ return *this;
+ }
+
+ const_fixed_iterator operator++(int) noexcept {
+ const const_fixed_iterator tmp{*this};
+ ++(*this);
+ return tmp;
+ }
+
+ const_fixed_iterator& operator--() noexcept {
+ m_data -= sizeof(value_type);
+ return *this;
+ }
+
+ const_fixed_iterator operator--(int) noexcept {
+ const const_fixed_iterator tmp{*this};
+ --(*this);
+ return tmp;
+ }
+
+ friend bool operator==(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
+ return lhs.m_data == rhs.m_data;
+ }
+
+ friend bool operator!=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
+ return !(lhs == rhs);
+ }
+
+ friend bool operator<(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
+ return lhs.m_data < rhs.m_data;
+ }
+
+ friend bool operator>(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
+ return rhs < lhs;
+ }
+
+ friend bool operator<=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
+ return !(lhs > rhs);
+ }
+
+ friend bool operator>=(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
+ return !(lhs < rhs);
+ }
+
+ const_fixed_iterator& operator+=(difference_type val) noexcept {
+ m_data += (sizeof(value_type) * val);
+ return *this;
+ }
+
+ friend const_fixed_iterator operator+(const_fixed_iterator lhs, difference_type rhs) noexcept {
+ const_fixed_iterator tmp{lhs};
+ tmp.m_data += (sizeof(value_type) * rhs);
+ return tmp;
+ }
+
+ friend const_fixed_iterator operator+(difference_type lhs, const_fixed_iterator rhs) noexcept {
+ const_fixed_iterator tmp{rhs};
+ tmp.m_data += (sizeof(value_type) * lhs);
+ return tmp;
+ }
+
+ const_fixed_iterator& operator-=(difference_type val) noexcept {
+ m_data -= (sizeof(value_type) * val);
+ return *this;
+ }
+
+ friend const_fixed_iterator operator-(const_fixed_iterator lhs, difference_type rhs) noexcept {
+ const_fixed_iterator tmp{lhs};
+ tmp.m_data -= (sizeof(value_type) * rhs);
+ return tmp;
+ }
+
+ friend difference_type operator-(const_fixed_iterator lhs, const_fixed_iterator rhs) noexcept {
+ return static_cast<difference_type>(lhs.m_data - rhs.m_data) / static_cast<difference_type>(sizeof(T));
+ }
+
+ value_type operator[](difference_type n) const noexcept {
+ return *(*this + n);
+ }
+
+ /// @endcond
+
+}; // class const_fixed_iterator
+
+/**
+ * A forward iterator used for accessing packed repeated varint fields
+ * (int32, uint32, int64, uint64, bool, enum).
+ */
+template <typename T>
+class const_varint_iterator {
+
+protected:
+
+ /// Pointer to current iterator position
+ const char* m_data = nullptr; // NOLINT(misc-non-private-member-variables-in-classes, cppcoreguidelines-non-private-member-variables-in-classes,-warnings-as-errors)
+
+ /// Pointer to end iterator position
+ const char* m_end = nullptr; // NOLINT(misc-non-private-member-variables-in-classes, cppcoreguidelines-non-private-member-variables-in-classes,-warnings-as-errors)
+
+public:
+
+ /// @cond usual iterator functions not documented
+
+ using iterator_category = std::forward_iterator_tag;
+ using value_type = T;
+ using difference_type = std::ptrdiff_t;
+ using pointer = value_type*;
+ using reference = value_type&;
+
+ static difference_type distance(const_varint_iterator begin, const_varint_iterator end) noexcept {
+ // The "distance" between default initialized const_varint_iterator's
+ // is always 0.
+ if (!begin.m_data) {
+ return 0;
+ }
+ // We know that each varint contains exactly one byte with the most
+ // significant bit not set. We can use this to quickly figure out
+ // how many varints there are without actually decoding the varints.
+ return std::count_if(begin.m_data, end.m_data, [](char c) noexcept {
+ return (static_cast<unsigned char>(c) & 0x80U) == 0;
+ });
+ }
+
+ const_varint_iterator() noexcept = default;
+
+ const_varint_iterator(const char* data, const char* end) noexcept :
+ m_data{data},
+ m_end{end} {
+ }
+
+ const_varint_iterator(const const_varint_iterator&) noexcept = default;
+ const_varint_iterator(const_varint_iterator&&) noexcept = default;
+
+ const_varint_iterator& operator=(const const_varint_iterator&) noexcept = default;
+ const_varint_iterator& operator=(const_varint_iterator&&) noexcept = default;
+
+ ~const_varint_iterator() noexcept = default;
+
+ value_type operator*() const {
+ protozero_assert(m_data);
+ const char* d = m_data; // will be thrown away
+ return static_cast<value_type>(decode_varint(&d, m_end));
+ }
+
+ const_varint_iterator& operator++() {
+ protozero_assert(m_data);
+ skip_varint(&m_data, m_end);
+ return *this;
+ }
+
+ const_varint_iterator operator++(int) {
+ protozero_assert(m_data);
+ const const_varint_iterator tmp{*this};
+ ++(*this);
+ return tmp;
+ }
+
+ bool operator==(const const_varint_iterator& rhs) const noexcept {
+ return m_data == rhs.m_data && m_end == rhs.m_end;
+ }
+
+ bool operator!=(const const_varint_iterator& rhs) const noexcept {
+ return !(*this == rhs);
+ }
+
+ /// @endcond
+
+}; // class const_varint_iterator
+
+/**
+ * A forward iterator used for accessing packed repeated svarint fields
+ * (sint32, sint64).
+ */
+template <typename T>
+class const_svarint_iterator : public const_varint_iterator<T> {
+
+public:
+
+ /// @cond usual iterator functions not documented
+
+ using iterator_category = std::forward_iterator_tag;
+ using value_type = T;
+ using difference_type = std::ptrdiff_t;
+ using pointer = value_type*;
+ using reference = value_type&;
+
+ const_svarint_iterator() noexcept :
+ const_varint_iterator<T>{} {
+ }
+
+ const_svarint_iterator(const char* data, const char* end) noexcept :
+ const_varint_iterator<T>{data, end} {
+ }
+
+ const_svarint_iterator(const const_svarint_iterator&) = default;
+ const_svarint_iterator(const_svarint_iterator&&) noexcept = default;
+
+ const_svarint_iterator& operator=(const const_svarint_iterator&) = default;
+ const_svarint_iterator& operator=(const_svarint_iterator&&) noexcept = default;
+
+ ~const_svarint_iterator() = default;
+
+ value_type operator*() const {
+ protozero_assert(this->m_data);
+ const char* d = this->m_data; // will be thrown away
+ return static_cast<value_type>(decode_zigzag64(decode_varint(&d, this->m_end)));
+ }
+
+ const_svarint_iterator& operator++() {
+ protozero_assert(this->m_data);
+ skip_varint(&this->m_data, this->m_end);
+ return *this;
+ }
+
+ const_svarint_iterator operator++(int) {
+ protozero_assert(this->m_data);
+ const const_svarint_iterator tmp{*this};
+ ++(*this);
+ return tmp;
+ }
+
+ /// @endcond
+
+}; // class const_svarint_iterator
+
+} // end namespace protozero
+
+namespace std {
+
+ // Specialize std::distance for all the protozero iterators. Because
+ // functions can't be partially specialized, we have to do this for
+ // every value_type we are using.
+
+ /// @cond individual overloads do not need to be documented
+
+ template <>
+ inline typename protozero::const_varint_iterator<int32_t>::difference_type
+ distance<protozero::const_varint_iterator<int32_t>>(protozero::const_varint_iterator<int32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
+ protozero::const_varint_iterator<int32_t> last) {
+ return protozero::const_varint_iterator<int32_t>::distance(first, last);
+ }
+
+ template <>
+ inline typename protozero::const_varint_iterator<int64_t>::difference_type
+ distance<protozero::const_varint_iterator<int64_t>>(protozero::const_varint_iterator<int64_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
+ protozero::const_varint_iterator<int64_t> last) {
+ return protozero::const_varint_iterator<int64_t>::distance(first, last);
+ }
+
+ template <>
+ inline typename protozero::const_varint_iterator<uint32_t>::difference_type
+ distance<protozero::const_varint_iterator<uint32_t>>(protozero::const_varint_iterator<uint32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
+ protozero::const_varint_iterator<uint32_t> last) {
+ return protozero::const_varint_iterator<uint32_t>::distance(first, last);
+ }
+
+ template <>
+ inline typename protozero::const_varint_iterator<uint64_t>::difference_type
+ distance<protozero::const_varint_iterator<uint64_t>>(protozero::const_varint_iterator<uint64_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
+ protozero::const_varint_iterator<uint64_t> last) {
+ return protozero::const_varint_iterator<uint64_t>::distance(first, last);
+ }
+
+ template <>
+ inline typename protozero::const_svarint_iterator<int32_t>::difference_type
+ distance<protozero::const_svarint_iterator<int32_t>>(protozero::const_svarint_iterator<int32_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
+ protozero::const_svarint_iterator<int32_t> last) {
+ return protozero::const_svarint_iterator<int32_t>::distance(first, last);
+ }
+
+ template <>
+ inline typename protozero::const_svarint_iterator<int64_t>::difference_type
+ distance<protozero::const_svarint_iterator<int64_t>>(protozero::const_svarint_iterator<int64_t> first, // NOLINT(readability-inconsistent-declaration-parameter-name)
+ protozero::const_svarint_iterator<int64_t> last) {
+ return protozero::const_svarint_iterator<int64_t>::distance(first, last);
+ }
+
+ /// @endcond
+
+} // end namespace std
+
+#endif // PROTOZERO_ITERATORS_HPP
diff --git a/thirdparty/protozero/include/protozero/pbf_builder.hpp b/thirdparty/protozero/include/protozero/pbf_builder.hpp
new file mode 100644
index 000000000..71a2dec2b
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/pbf_builder.hpp
@@ -0,0 +1,32 @@
+#ifndef PROTOZERO_PBF_BUILDER_HPP
+#define PROTOZERO_PBF_BUILDER_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file pbf_builder.hpp
+ *
+ * @brief Contains the pbf_builder template class.
+ */
+
+#include "basic_pbf_builder.hpp"
+#include "pbf_writer.hpp"
+
+#include <string>
+
+namespace protozero {
+
+/// Specialization of basic_pbf_builder using std::string as buffer type.
+template <typename T>
+using pbf_builder = basic_pbf_builder<std::string, T>;
+
+} // end namespace protozero
+
+#endif // PROTOZERO_PBF_BUILDER_HPP
diff --git a/thirdparty/protozero/include/protozero/pbf_message.hpp b/thirdparty/protozero/include/protozero/pbf_message.hpp
new file mode 100644
index 000000000..393078b9b
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/pbf_message.hpp
@@ -0,0 +1,184 @@
+#ifndef PROTOZERO_PBF_MESSAGE_HPP
+#define PROTOZERO_PBF_MESSAGE_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file pbf_message.hpp
+ *
+ * @brief Contains the pbf_message template class.
+ */
+
+#include "pbf_reader.hpp"
+#include "types.hpp"
+
+#include <type_traits>
+
+namespace protozero {
+
+/**
+ * This class represents a protobuf message. Either a top-level message or
+ * a nested sub-message. Top-level messages can be created from any buffer
+ * with a pointer and length:
+ *
+ * @code
+ * enum class Message : protozero::pbf_tag_type {
+ * ...
+ * };
+ *
+ * std::string buffer;
+ * // fill buffer...
+ * pbf_message<Message> message{buffer.data(), buffer.size()};
+ * @endcode
+ *
+ * Sub-messages are created using get_message():
+ *
+ * @code
+ * enum class SubMessage : protozero::pbf_tag_type {
+ * ...
+ * };
+ *
+ * pbf_message<Message> message{...};
+ * message.next();
+ * pbf_message<SubMessage> submessage = message.get_message();
+ * @endcode
+ *
+ * All methods of the pbf_message class except get_bytes() and get_string()
+ * provide the strong exception guarantee, ie they either succeed or do not
+ * change the pbf_message object they are called on. Use the get_data() method
+ * instead of get_bytes() or get_string(), if you need this guarantee.
+ *
+ * This template class is based on the pbf_reader class and has all the same
+ * methods. The difference is that whereever the pbf_reader class takes an
+ * integer tag, this template class takes a tag of the template type T.
+ *
+ * Read the tutorial to understand how this class is used.
+ */
+template <typename T>
+class pbf_message : public pbf_reader {
+
+ static_assert(std::is_same<pbf_tag_type, std::underlying_type_t<T>>::value,
+ "T must be enum with underlying type protozero::pbf_tag_type");
+
+public:
+
+ /// The type of messages this class will read.
+ using enum_type = T;
+
+ /**
+ * Construct a pbf_message. All arguments are forwarded to the pbf_reader
+ * parent class.
+ */
+ template <typename... Args>
+ pbf_message(Args&&... args) noexcept : // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
+ pbf_reader{std::forward<Args>(args)...} {
+ }
+
+ /**
+ * Set next field in the message as the current field. This is usually
+ * called in a while loop:
+ *
+ * @code
+ * pbf_message<...> message(...);
+ * while (message.next()) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * @returns `true` if there is a next field, `false` if not.
+ * @pre There must be no current field.
+ * @post If it returns `true` there is a current field now.
+ */
+ bool next() {
+ return pbf_reader::next();
+ }
+
+ /**
+ * Set next field with given tag in the message as the current field.
+ * Fields with other tags are skipped. This is usually called in a while
+ * loop for repeated fields:
+ *
+ * @code
+ * pbf_message<Example1> message{...};
+ * while (message.next(Example1::repeated_fixed64_r)) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * or you can call it just once to get the one field with this tag:
+ *
+ * @code
+ * pbf_message<Example1> message{...};
+ * if (message.next(Example1::required_uint32_x)) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * Note that this will not check the wire type. The two-argument version
+ * of this function will also check the wire type.
+ *
+ * @returns `true` if there is a next field with this tag.
+ * @pre There must be no current field.
+ * @post If it returns `true` there is a current field now with the given tag.
+ */
+ bool next(T next_tag) {
+ return pbf_reader::next(pbf_tag_type(next_tag));
+ }
+
+ /**
+ * Set next field with given tag and wire type in the message as the
+ * current field. Fields with other tags are skipped. This is usually
+ * called in a while loop for repeated fields:
+ *
+ * @code
+ * pbf_message<Example1> message{...};
+ * while (message.next(Example1::repeated_fixed64_r, pbf_wire_type::varint)) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * or you can call it just once to get the one field with this tag:
+ *
+ * @code
+ * pbf_message<Example1> message{...};
+ * if (message.next(Example1::required_uint32_x, pbf_wire_type::varint)) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * Note that this will also check the wire type. The one-argument version
+ * of this function will not check the wire type.
+ *
+ * @returns `true` if there is a next field with this tag.
+ * @pre There must be no current field.
+ * @post If it returns `true` there is a current field now with the given tag.
+ */
+ bool next(T next_tag, pbf_wire_type type) {
+ return pbf_reader::next(pbf_tag_type(next_tag), type);
+ }
+
+ /**
+ * The tag of the current field. The tag is the enum value for the field
+ * number from the description in the .proto file.
+ *
+ * Call next() before calling this function to set the current field.
+ *
+ * @returns tag of the current field.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ */
+ T tag() const noexcept {
+ return T(pbf_reader::tag());
+ }
+
+}; // class pbf_message
+
+} // end namespace protozero
+
+#endif // PROTOZERO_PBF_MESSAGE_HPP
diff --git a/thirdparty/protozero/include/protozero/pbf_reader.hpp b/thirdparty/protozero/include/protozero/pbf_reader.hpp
new file mode 100644
index 000000000..fab4dceb7
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/pbf_reader.hpp
@@ -0,0 +1,977 @@
+#ifndef PROTOZERO_PBF_READER_HPP
+#define PROTOZERO_PBF_READER_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file pbf_reader.hpp
+ *
+ * @brief Contains the pbf_reader class.
+ */
+
+#include "config.hpp"
+#include "data_view.hpp"
+#include "exception.hpp"
+#include "iterators.hpp"
+#include "types.hpp"
+#include "varint.hpp"
+
+#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
+# include <protozero/byteswap.hpp>
+#endif
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <utility>
+
+namespace protozero {
+
+/**
+ * This class represents a protobuf message. Either a top-level message or
+ * a nested sub-message. Top-level messages can be created from any buffer
+ * with a pointer and length:
+ *
+ * @code
+ * std::string buffer;
+ * // fill buffer...
+ * pbf_reader message{buffer.data(), buffer.size()};
+ * @endcode
+ *
+ * Sub-messages are created using get_message():
+ *
+ * @code
+ * pbf_reader message{...};
+ * message.next();
+ * pbf_reader submessage = message.get_message();
+ * @endcode
+ *
+ * All methods of the pbf_reader class except get_bytes() and get_string()
+ * provide the strong exception guarantee, ie they either succeed or do not
+ * change the pbf_reader object they are called on. Use the get_view() method
+ * instead of get_bytes() or get_string(), if you need this guarantee.
+ */
+class pbf_reader {
+
+ // A pointer to the next unread data.
+ const char* m_data = nullptr;
+
+ // A pointer to one past the end of data.
+ const char* m_end = nullptr;
+
+ // The wire type of the current field.
+ pbf_wire_type m_wire_type = pbf_wire_type::unknown;
+
+ // The tag of the current field.
+ pbf_tag_type m_tag = 0;
+
+ template <typename T>
+ T get_fixed() {
+ T result;
+ const char* tmp_data = m_data;
+ skip_bytes(sizeof(T));
+ std::memcpy(&result, tmp_data, sizeof(T));
+#if PROTOZERO_BYTE_ORDER != PROTOZERO_LITTLE_ENDIAN
+ byteswap_inplace(&result);
+#endif
+ return result;
+ }
+
+ template <typename T>
+ iterator_range<const_fixed_iterator<T>> packed_fixed() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ const auto len = get_len_and_skip();
+ if (len % sizeof(T) != 0) {
+ throw invalid_length_exception{};
+ }
+ return {const_fixed_iterator<T>(m_data - len),
+ const_fixed_iterator<T>(m_data)};
+ }
+
+ template <typename T>
+ T get_varint() {
+ const auto val = static_cast<T>(decode_varint(&m_data, m_end));
+ return val;
+ }
+
+ template <typename T>
+ T get_svarint() {
+ protozero_assert((has_wire_type(pbf_wire_type::varint) || has_wire_type(pbf_wire_type::length_delimited)) && "not a varint");
+ return static_cast<T>(decode_zigzag64(decode_varint(&m_data, m_end)));
+ }
+
+ pbf_length_type get_length() {
+ return get_varint<pbf_length_type>();
+ }
+
+ void skip_bytes(pbf_length_type len) {
+ if (m_end - m_data < static_cast<ptrdiff_t>(len)) {
+ throw end_of_buffer_exception{};
+ }
+ m_data += len;
+
+#ifndef NDEBUG
+ // In debug builds reset the tag to zero so that we can detect (some)
+ // wrong code.
+ m_tag = 0;
+#endif
+ }
+
+ pbf_length_type get_len_and_skip() {
+ const auto len = get_length();
+ skip_bytes(len);
+ return len;
+ }
+
+ template <typename T>
+ iterator_range<T> get_packed() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ const auto len = get_len_and_skip();
+ return {T{m_data - len, m_data},
+ T{m_data, m_data}};
+ }
+
+public:
+
+ /**
+ * Construct a pbf_reader message from a data_view. The pointer from the
+ * data_view will be stored inside the pbf_reader object, no data is
+ * copied. So you must make sure the view stays valid as long as the
+ * pbf_reader object is used.
+ *
+ * The buffer must contain a complete protobuf message.
+ *
+ * @post There is no current field.
+ */
+ explicit pbf_reader(const data_view& view) noexcept
+ : m_data{view.data()},
+ m_end{view.data() + view.size()} {
+ }
+
+ /**
+ * Construct a pbf_reader message from a data pointer and a length. The
+ * pointer will be stored inside the pbf_reader object, no data is copied.
+ * So you must make sure the buffer stays valid as long as the pbf_reader
+ * object is used.
+ *
+ * The buffer must contain a complete protobuf message.
+ *
+ * @post There is no current field.
+ */
+ pbf_reader(const char* data, std::size_t size) noexcept
+ : m_data{data},
+ m_end{data + size} {
+ }
+
+#ifndef PROTOZERO_STRICT_API
+ /**
+ * Construct a pbf_reader message from a data pointer and a length. The
+ * pointer will be stored inside the pbf_reader object, no data is copied.
+ * So you must make sure the buffer stays valid as long as the pbf_reader
+ * object is used.
+ *
+ * The buffer must contain a complete protobuf message.
+ *
+ * @post There is no current field.
+ * @deprecated Use one of the other constructors.
+ */
+ explicit pbf_reader(const std::pair<const char*, std::size_t>& data) noexcept
+ : m_data{data.first},
+ m_end{data.first + data.second} {
+ }
+#endif
+
+ /**
+ * Construct a pbf_reader message from a std::string. A pointer to the
+ * string internals will be stored inside the pbf_reader object, no data
+ * is copied. So you must make sure the string is unchanged as long as the
+ * pbf_reader object is used.
+ *
+ * The string must contain a complete protobuf message.
+ *
+ * @post There is no current field.
+ */
+ explicit pbf_reader(const std::string& data) noexcept
+ : m_data{data.data()},
+ m_end{data.data() + data.size()} {
+ }
+
+ /**
+ * pbf_reader can be default constructed and behaves like it has an empty
+ * buffer.
+ */
+ pbf_reader() noexcept = default;
+
+ /// pbf_reader messages can be copied trivially.
+ pbf_reader(const pbf_reader&) noexcept = default;
+
+ /// pbf_reader messages can be moved trivially.
+ pbf_reader(pbf_reader&&) noexcept = default;
+
+ /// pbf_reader messages can be copied trivially.
+ pbf_reader& operator=(const pbf_reader& other) noexcept = default;
+
+ /// pbf_reader messages can be moved trivially.
+ pbf_reader& operator=(pbf_reader&& other) noexcept = default;
+
+ ~pbf_reader() = default;
+
+ /**
+ * Swap the contents of this object with the other.
+ *
+ * @param other Other object to swap data with.
+ */
+ void swap(pbf_reader& other) noexcept {
+ using std::swap;
+ swap(m_data, other.m_data);
+ swap(m_end, other.m_end);
+ swap(m_wire_type, other.m_wire_type);
+ swap(m_tag, other.m_tag);
+ }
+
+ /**
+ * In a boolean context the pbf_reader class evaluates to `true` if there
+ * are still fields available and to `false` if the last field has been
+ * read.
+ */
+ operator bool() const noexcept { // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
+ return m_data != m_end;
+ }
+
+ /**
+ * Get a view of the not yet read data.
+ */
+ data_view data() const noexcept {
+ return {m_data, static_cast<std::size_t>(m_end - m_data)};
+ }
+
+ /**
+ * Return the length in bytes of the current message. If you have
+ * already called next() and/or any of the get_*() functions, this will
+ * return the remaining length.
+ *
+ * This can, for instance, be used to estimate the space needed for a
+ * buffer. Of course you have to know reasonably well what data to expect
+ * and how it is encoded for this number to have any meaning.
+ */
+ std::size_t length() const noexcept {
+ return static_cast<std::size_t>(m_end - m_data);
+ }
+
+ /**
+ * Set next field in the message as the current field. This is usually
+ * called in a while loop:
+ *
+ * @code
+ * pbf_reader message(...);
+ * while (message.next()) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * @returns `true` if there is a next field, `false` if not.
+ * @pre There must be no current field.
+ * @post If it returns `true` there is a current field now.
+ */
+ bool next() {
+ if (m_data == m_end) {
+ return false;
+ }
+
+ const auto value = get_varint<uint32_t>();
+ m_tag = static_cast<pbf_tag_type>(value >> 3U);
+
+ // tags 0 and 19000 to 19999 are not allowed as per
+ // https://developers.google.com/protocol-buffers/docs/proto#assigning-tags
+ if (m_tag == 0 || (m_tag >= 19000 && m_tag <= 19999)) {
+ throw invalid_tag_exception{};
+ }
+
+ m_wire_type = pbf_wire_type(value & 0x07U);
+ switch (m_wire_type) {
+ case pbf_wire_type::varint:
+ case pbf_wire_type::fixed64:
+ case pbf_wire_type::length_delimited:
+ case pbf_wire_type::fixed32:
+ break;
+ default:
+ throw unknown_pbf_wire_type_exception{};
+ }
+
+ return true;
+ }
+
+ /**
+ * Set next field with given tag in the message as the current field.
+ * Fields with other tags are skipped. This is usually called in a while
+ * loop for repeated fields:
+ *
+ * @code
+ * pbf_reader message{...};
+ * while (message.next(17)) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * or you can call it just once to get the one field with this tag:
+ *
+ * @code
+ * pbf_reader message{...};
+ * if (message.next(17)) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * Note that this will not check the wire type. The two-argument version
+ * of this function will also check the wire type.
+ *
+ * @returns `true` if there is a next field with this tag.
+ * @pre There must be no current field.
+ * @post If it returns `true` there is a current field now with the given tag.
+ */
+ bool next(pbf_tag_type next_tag) {
+ while (next()) {
+ if (m_tag == next_tag) {
+ return true;
+ }
+ skip();
+ }
+ return false;
+ }
+
+ /**
+ * Set next field with given tag and wire type in the message as the
+ * current field. Fields with other tags are skipped. This is usually
+ * called in a while loop for repeated fields:
+ *
+ * @code
+ * pbf_reader message{...};
+ * while (message.next(17, pbf_wire_type::varint)) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * or you can call it just once to get the one field with this tag:
+ *
+ * @code
+ * pbf_reader message{...};
+ * if (message.next(17, pbf_wire_type::varint)) {
+ * // handle field
+ * }
+ * @endcode
+ *
+ * Note that this will also check the wire type. The one-argument version
+ * of this function will not check the wire type.
+ *
+ * @returns `true` if there is a next field with this tag.
+ * @pre There must be no current field.
+ * @post If it returns `true` there is a current field now with the given tag.
+ */
+ bool next(pbf_tag_type next_tag, pbf_wire_type type) {
+ while (next()) {
+ if (m_tag == next_tag && m_wire_type == type) {
+ return true;
+ }
+ skip();
+ }
+ return false;
+ }
+
+ /**
+ * The tag of the current field. The tag is the field number from the
+ * description in the .proto file.
+ *
+ * Call next() before calling this function to set the current field.
+ *
+ * @returns tag of the current field.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ */
+ pbf_tag_type tag() const noexcept {
+ return m_tag;
+ }
+
+ /**
+ * Get the wire type of the current field. The wire types are:
+ *
+ * * 0 - varint
+ * * 1 - 64 bit
+ * * 2 - length-delimited
+ * * 5 - 32 bit
+ *
+ * All other types are illegal.
+ *
+ * Call next() before calling this function to set the current field.
+ *
+ * @returns wire type of the current field.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ */
+ pbf_wire_type wire_type() const noexcept {
+ return m_wire_type;
+ }
+
+ /**
+ * Get the tag and wire type of the current field in one integer suitable
+ * for comparison with a switch statement.
+ *
+ * Use it like this:
+ *
+ * @code
+ * pbf_reader message{...};
+ * while (message.next()) {
+ * switch (message.tag_and_type()) {
+ * case tag_and_type(17, pbf_wire_type::length_delimited):
+ * ....
+ * break;
+ * case tag_and_type(21, pbf_wire_type::varint):
+ * ....
+ * break;
+ * default:
+ * message.skip();
+ * }
+ * }
+ * @endcode
+ */
+ uint32_t tag_and_type() const noexcept {
+ return protozero::tag_and_type(tag(), wire_type());
+ }
+
+ /**
+ * Check the wire type of the current field.
+ *
+ * @returns `true` if the current field has the given wire type.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ */
+ bool has_wire_type(pbf_wire_type type) const noexcept {
+ return wire_type() == type;
+ }
+
+ /**
+ * Consume the current field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @post The current field was consumed and there is no current field now.
+ */
+ void skip() {
+ protozero_assert(tag() != 0 && "call next() before calling skip()");
+ switch (wire_type()) {
+ case pbf_wire_type::varint:
+ skip_varint(&m_data, m_end);
+ break;
+ case pbf_wire_type::fixed64:
+ skip_bytes(8);
+ break;
+ case pbf_wire_type::length_delimited:
+ skip_bytes(get_length());
+ break;
+ case pbf_wire_type::fixed32:
+ skip_bytes(4);
+ break;
+ default:
+ break;
+ }
+ }
+
+ ///@{
+ /**
+ * @name Scalar field accessor functions
+ */
+
+ /**
+ * Consume and return value of current "bool" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "bool".
+ * @post The current field was consumed and there is no current field now.
+ */
+ bool get_bool() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
+ const char* value = m_data;
+ skip_varint(&m_data, m_end);
+ return *value != 0;
+ }
+
+ /**
+ * Consume and return value of current "enum" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "enum".
+ * @post The current field was consumed and there is no current field now.
+ */
+ int32_t get_enum() {
+ protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
+ return get_varint<int32_t>();
+ }
+
+ /**
+ * Consume and return value of current "int32" varint field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "int32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ int32_t get_int32() {
+ protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
+ return get_varint<int32_t>();
+ }
+
+ /**
+ * Consume and return value of current "sint32" varint field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "sint32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ int32_t get_sint32() {
+ protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
+ return get_svarint<int32_t>();
+ }
+
+ /**
+ * Consume and return value of current "uint32" varint field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "uint32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ uint32_t get_uint32() {
+ protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
+ return get_varint<uint32_t>();
+ }
+
+ /**
+ * Consume and return value of current "int64" varint field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "int64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ int64_t get_int64() {
+ protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
+ return get_varint<int64_t>();
+ }
+
+ /**
+ * Consume and return value of current "sint64" varint field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "sint64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ int64_t get_sint64() {
+ protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
+ return get_svarint<int64_t>();
+ }
+
+ /**
+ * Consume and return value of current "uint64" varint field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "uint64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ uint64_t get_uint64() {
+ protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint");
+ return get_varint<uint64_t>();
+ }
+
+ /**
+ * Consume and return value of current "fixed32" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "fixed32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ uint32_t get_fixed32() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed");
+ return get_fixed<uint32_t>();
+ }
+
+ /**
+ * Consume and return value of current "sfixed32" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "sfixed32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ int32_t get_sfixed32() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed");
+ return get_fixed<int32_t>();
+ }
+
+ /**
+ * Consume and return value of current "fixed64" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "fixed64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ uint64_t get_fixed64() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed");
+ return get_fixed<uint64_t>();
+ }
+
+ /**
+ * Consume and return value of current "sfixed64" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "sfixed64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ int64_t get_sfixed64() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed");
+ return get_fixed<int64_t>();
+ }
+
+ /**
+ * Consume and return value of current "float" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "float".
+ * @post The current field was consumed and there is no current field now.
+ */
+ float get_float() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed");
+ return get_fixed<float>();
+ }
+
+ /**
+ * Consume and return value of current "double" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "double".
+ * @post The current field was consumed and there is no current field now.
+ */
+ double get_double() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed");
+ return get_fixed<double>();
+ }
+
+ /**
+ * Consume and return value of current "bytes", "string", or "message"
+ * field.
+ *
+ * @returns A data_view object.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "bytes", "string", or "message".
+ * @post The current field was consumed and there is no current field now.
+ */
+ data_view get_view() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message");
+ const auto len = get_len_and_skip();
+ return {m_data - len, len};
+ }
+
+#ifndef PROTOZERO_STRICT_API
+ /**
+ * Consume and return value of current "bytes" or "string" field.
+ *
+ * @returns A pair with a pointer to the data and the length of the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "bytes" or "string".
+ * @post The current field was consumed and there is no current field now.
+ */
+ std::pair<const char*, pbf_length_type> get_data() {
+ protozero_assert(tag() != 0 && "call next() before accessing field value");
+ protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message");
+ const auto len = get_len_and_skip();
+ return {m_data - len, len};
+ }
+#endif
+
+ /**
+ * Consume and return value of current "bytes" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "bytes".
+ * @post The current field was consumed and there is no current field now.
+ */
+ std::string get_bytes() {
+ return std::string(get_view());
+ }
+
+ /**
+ * Consume and return value of current "string" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "string".
+ * @post The current field was consumed and there is no current field now.
+ */
+ std::string get_string() {
+ return std::string(get_view());
+ }
+
+ /**
+ * Consume and return value of current "message" field.
+ *
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "message".
+ * @post The current field was consumed and there is no current field now.
+ */
+ pbf_reader get_message() {
+ return pbf_reader{get_view()};
+ }
+
+ ///@}
+
+ /// Forward iterator for iterating over bool (int32 varint) values.
+ using const_bool_iterator = const_varint_iterator< int32_t>;
+
+ /// Forward iterator for iterating over enum (int32 varint) values.
+ using const_enum_iterator = const_varint_iterator< int32_t>;
+
+ /// Forward iterator for iterating over int32 (varint) values.
+ using const_int32_iterator = const_varint_iterator< int32_t>;
+
+ /// Forward iterator for iterating over sint32 (varint) values.
+ using const_sint32_iterator = const_svarint_iterator<int32_t>;
+
+ /// Forward iterator for iterating over uint32 (varint) values.
+ using const_uint32_iterator = const_varint_iterator<uint32_t>;
+
+ /// Forward iterator for iterating over int64 (varint) values.
+ using const_int64_iterator = const_varint_iterator< int64_t>;
+
+ /// Forward iterator for iterating over sint64 (varint) values.
+ using const_sint64_iterator = const_svarint_iterator<int64_t>;
+
+ /// Forward iterator for iterating over uint64 (varint) values.
+ using const_uint64_iterator = const_varint_iterator<uint64_t>;
+
+ /// Forward iterator for iterating over fixed32 values.
+ using const_fixed32_iterator = const_fixed_iterator<uint32_t>;
+
+ /// Forward iterator for iterating over sfixed32 values.
+ using const_sfixed32_iterator = const_fixed_iterator<int32_t>;
+
+ /// Forward iterator for iterating over fixed64 values.
+ using const_fixed64_iterator = const_fixed_iterator<uint64_t>;
+
+ /// Forward iterator for iterating over sfixed64 values.
+ using const_sfixed64_iterator = const_fixed_iterator<int64_t>;
+
+ /// Forward iterator for iterating over float values.
+ using const_float_iterator = const_fixed_iterator<float>;
+
+ /// Forward iterator for iterating over double values.
+ using const_double_iterator = const_fixed_iterator<double>;
+
+ ///@{
+ /**
+ * @name Repeated packed field accessor functions
+ */
+
+ /**
+ * Consume current "repeated packed bool" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed bool".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_bool_iterator> get_packed_bool() {
+ return get_packed<pbf_reader::const_bool_iterator>();
+ }
+
+ /**
+ * Consume current "repeated packed enum" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed enum".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_enum_iterator> get_packed_enum() {
+ return get_packed<pbf_reader::const_enum_iterator>();
+ }
+
+ /**
+ * Consume current "repeated packed int32" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed int32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_int32_iterator> get_packed_int32() {
+ return get_packed<pbf_reader::const_int32_iterator>();
+ }
+
+ /**
+ * Consume current "repeated packed sint32" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed sint32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_sint32_iterator> get_packed_sint32() {
+ return get_packed<pbf_reader::const_sint32_iterator>();
+ }
+
+ /**
+ * Consume current "repeated packed uint32" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed uint32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_uint32_iterator> get_packed_uint32() {
+ return get_packed<pbf_reader::const_uint32_iterator>();
+ }
+
+ /**
+ * Consume current "repeated packed int64" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed int64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_int64_iterator> get_packed_int64() {
+ return get_packed<pbf_reader::const_int64_iterator>();
+ }
+
+ /**
+ * Consume current "repeated packed sint64" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed sint64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_sint64_iterator> get_packed_sint64() {
+ return get_packed<pbf_reader::const_sint64_iterator>();
+ }
+
+ /**
+ * Consume current "repeated packed uint64" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed uint64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_uint64_iterator> get_packed_uint64() {
+ return get_packed<pbf_reader::const_uint64_iterator>();
+ }
+
+ /**
+ * Consume current "repeated packed fixed32" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed fixed32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_fixed32_iterator> get_packed_fixed32() {
+ return packed_fixed<uint32_t>();
+ }
+
+ /**
+ * Consume current "repeated packed sfixed32" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed sfixed32".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_sfixed32_iterator> get_packed_sfixed32() {
+ return packed_fixed<int32_t>();
+ }
+
+ /**
+ * Consume current "repeated packed fixed64" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed fixed64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_fixed64_iterator> get_packed_fixed64() {
+ return packed_fixed<uint64_t>();
+ }
+
+ /**
+ * Consume current "repeated packed sfixed64" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed sfixed64".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_sfixed64_iterator> get_packed_sfixed64() {
+ return packed_fixed<int64_t>();
+ }
+
+ /**
+ * Consume current "repeated packed float" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed float".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_float_iterator> get_packed_float() {
+ return packed_fixed<float>();
+ }
+
+ /**
+ * Consume current "repeated packed double" field.
+ *
+ * @returns a pair of iterators to the beginning and one past the end of
+ * the data.
+ * @pre There must be a current field (ie. next() must have returned `true`).
+ * @pre The current field must be of type "repeated packed double".
+ * @post The current field was consumed and there is no current field now.
+ */
+ iterator_range<pbf_reader::const_double_iterator> get_packed_double() {
+ return packed_fixed<double>();
+ }
+
+ ///@}
+
+}; // class pbf_reader
+
+/**
+ * Swap two pbf_reader objects.
+ *
+ * @param lhs First object.
+ * @param rhs Second object.
+ */
+inline void swap(pbf_reader& lhs, pbf_reader& rhs) noexcept {
+ lhs.swap(rhs);
+}
+
+} // end namespace protozero
+
+#endif // PROTOZERO_PBF_READER_HPP
diff --git a/thirdparty/protozero/include/protozero/pbf_writer.hpp b/thirdparty/protozero/include/protozero/pbf_writer.hpp
new file mode 100644
index 000000000..9a07bd5b9
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/pbf_writer.hpp
@@ -0,0 +1,76 @@
+#ifndef PROTOZERO_PBF_WRITER_HPP
+#define PROTOZERO_PBF_WRITER_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file pbf_writer.hpp
+ *
+ * @brief Contains the pbf_writer class.
+ */
+
+#include "basic_pbf_writer.hpp"
+#include "buffer_string.hpp"
+
+#include <cstdint>
+#include <string>
+
+namespace protozero {
+
+/**
+ * Specialization of basic_pbf_writer using std::string as buffer type.
+ */
+using pbf_writer = basic_pbf_writer<std::string>;
+
+/// Class for generating packed repeated bool fields.
+using packed_field_bool = detail::packed_field_varint<std::string, bool>;
+
+/// Class for generating packed repeated enum fields.
+using packed_field_enum = detail::packed_field_varint<std::string, int32_t>;
+
+/// Class for generating packed repeated int32 fields.
+using packed_field_int32 = detail::packed_field_varint<std::string, int32_t>;
+
+/// Class for generating packed repeated sint32 fields.
+using packed_field_sint32 = detail::packed_field_svarint<std::string, int32_t>;
+
+/// Class for generating packed repeated uint32 fields.
+using packed_field_uint32 = detail::packed_field_varint<std::string, uint32_t>;
+
+/// Class for generating packed repeated int64 fields.
+using packed_field_int64 = detail::packed_field_varint<std::string, int64_t>;
+
+/// Class for generating packed repeated sint64 fields.
+using packed_field_sint64 = detail::packed_field_svarint<std::string, int64_t>;
+
+/// Class for generating packed repeated uint64 fields.
+using packed_field_uint64 = detail::packed_field_varint<std::string, uint64_t>;
+
+/// Class for generating packed repeated fixed32 fields.
+using packed_field_fixed32 = detail::packed_field_fixed<std::string, uint32_t>;
+
+/// Class for generating packed repeated sfixed32 fields.
+using packed_field_sfixed32 = detail::packed_field_fixed<std::string, int32_t>;
+
+/// Class for generating packed repeated fixed64 fields.
+using packed_field_fixed64 = detail::packed_field_fixed<std::string, uint64_t>;
+
+/// Class for generating packed repeated sfixed64 fields.
+using packed_field_sfixed64 = detail::packed_field_fixed<std::string, int64_t>;
+
+/// Class for generating packed repeated float fields.
+using packed_field_float = detail::packed_field_fixed<std::string, float>;
+
+/// Class for generating packed repeated double fields.
+using packed_field_double = detail::packed_field_fixed<std::string, double>;
+
+} // end namespace protozero
+
+#endif // PROTOZERO_PBF_WRITER_HPP
diff --git a/thirdparty/protozero/include/protozero/types.hpp b/thirdparty/protozero/include/protozero/types.hpp
new file mode 100644
index 000000000..ad9c362ff
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/types.hpp
@@ -0,0 +1,66 @@
+#ifndef PROTOZERO_TYPES_HPP
+#define PROTOZERO_TYPES_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file types.hpp
+ *
+ * @brief Contains the declaration of low-level types used in the pbf format.
+ */
+
+#include "config.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <utility>
+
+namespace protozero {
+
+/**
+ * The type used for field tags (field numbers).
+ */
+using pbf_tag_type = uint32_t;
+
+/**
+ * The type used to encode type information.
+ * See the table on
+ * https://developers.google.com/protocol-buffers/docs/encoding
+ */
+enum class pbf_wire_type : uint32_t {
+ varint = 0, // int32/64, uint32/64, sint32/64, bool, enum
+ fixed64 = 1, // fixed64, sfixed64, double
+ length_delimited = 2, // string, bytes, nested messages, packed repeated fields
+ fixed32 = 5, // fixed32, sfixed32, float
+ unknown = 99 // used for default setting in this library
+};
+
+/**
+ * Get the tag and wire type of the current field in one integer suitable
+ * for comparison with a switch statement.
+ *
+ * See pbf_reader.tag_and_type() for an example how to use this.
+ */
+template <typename T>
+constexpr uint32_t tag_and_type(T tag, pbf_wire_type wire_type) noexcept {
+ return (static_cast<uint32_t>(static_cast<pbf_tag_type>(tag)) << 3U) | static_cast<uint32_t>(wire_type);
+}
+
+/**
+ * The type used for length values, such as the length of a field.
+ */
+using pbf_length_type = uint32_t;
+
+} // end namespace protozero
+
+#endif // PROTOZERO_TYPES_HPP
diff --git a/thirdparty/protozero/include/protozero/varint.hpp b/thirdparty/protozero/include/protozero/varint.hpp
new file mode 100644
index 000000000..5c7fcec75
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/varint.hpp
@@ -0,0 +1,244 @@
+#ifndef PROTOZERO_VARINT_HPP
+#define PROTOZERO_VARINT_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file varint.hpp
+ *
+ * @brief Contains low-level varint and zigzag encoding and decoding functions.
+ */
+
+#include "buffer_tmpl.hpp"
+#include "exception.hpp"
+
+#include <cstdint>
+
+namespace protozero {
+
+/**
+ * The maximum length of a 64 bit varint.
+ */
+constexpr const int8_t max_varint_length = (sizeof(uint64_t) * 8 / 7) + 1;
+
+namespace detail {
+
+ // from https://github.com/facebook/folly/blob/master/folly/Varint.h
+ inline uint64_t decode_varint_impl(const char** data, const char* end) {
+ const auto* begin = reinterpret_cast<const int8_t*>(*data);
+ const auto* iend = reinterpret_cast<const int8_t*>(end);
+ const int8_t* p = begin;
+ uint64_t val = 0;
+
+ if (iend - begin >= max_varint_length) { // fast path
+ do {
+ int64_t b = *p++;
+ val = ((static_cast<uint64_t>(b) & 0x7fU) ); if (b >= 0) { break; }
+ b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 7U); if (b >= 0) { break; }
+ b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 14U); if (b >= 0) { break; }
+ b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 21U); if (b >= 0) { break; }
+ b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 28U); if (b >= 0) { break; }
+ b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 35U); if (b >= 0) { break; }
+ b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 42U); if (b >= 0) { break; }
+ b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 49U); if (b >= 0) { break; }
+ b = *p++; val |= ((static_cast<uint64_t>(b) & 0x7fU) << 56U); if (b >= 0) { break; }
+ b = *p++; val |= ((static_cast<uint64_t>(b) & 0x01U) << 63U); if (b >= 0) { break; }
+ throw varint_too_long_exception{};
+ } while (false);
+ } else {
+ unsigned int shift = 0;
+ while (p != iend && *p < 0) {
+ val |= (static_cast<uint64_t>(*p++) & 0x7fU) << shift;
+ shift += 7;
+ }
+ if (p == iend) {
+ throw end_of_buffer_exception{};
+ }
+ val |= static_cast<uint64_t>(*p++) << shift;
+ }
+
+ *data = reinterpret_cast<const char*>(p);
+ return val;
+ }
+
+} // end namespace detail
+
+/**
+ * Decode a 64 bit varint.
+ *
+ * Strong exception guarantee: if there is an exception the data pointer will
+ * not be changed.
+ *
+ * @param[in,out] data Pointer to pointer to the input data. After the function
+ * returns this will point to the next data to be read.
+ * @param[in] end Pointer one past the end of the input data.
+ * @returns The decoded integer
+ * @throws varint_too_long_exception if the varint is longer then the maximum
+ * length that would fit in a 64 bit int. Usually this means your data
+ * is corrupted or you are trying to read something as a varint that
+ * isn't.
+ * @throws end_of_buffer_exception if the *end* of the buffer was reached
+ * before the end of the varint.
+ */
+inline uint64_t decode_varint(const char** data, const char* end) {
+ // If this is a one-byte varint, decode it here.
+ if (end != *data && ((static_cast<uint64_t>(**data) & 0x80U) == 0)) {
+ const auto val = static_cast<uint64_t>(**data);
+ ++(*data);
+ return val;
+ }
+ // If this varint is more than one byte, defer to complete implementation.
+ return detail::decode_varint_impl(data, end);
+}
+
+/**
+ * Skip over a varint.
+ *
+ * Strong exception guarantee: if there is an exception the data pointer will
+ * not be changed.
+ *
+ * @param[in,out] data Pointer to pointer to the input data. After the function
+ * returns this will point to the next data to be read.
+ * @param[in] end Pointer one past the end of the input data.
+ * @throws end_of_buffer_exception if the *end* of the buffer was reached
+ * before the end of the varint.
+ */
+inline void skip_varint(const char** data, const char* end) {
+ const auto* begin = reinterpret_cast<const int8_t*>(*data);
+ const auto* iend = reinterpret_cast<const int8_t*>(end);
+ const int8_t* p = begin;
+
+ while (p != iend && *p < 0) {
+ ++p;
+ }
+
+ if (p - begin >= max_varint_length) {
+ throw varint_too_long_exception{};
+ }
+
+ if (p == iend) {
+ throw end_of_buffer_exception{};
+ }
+
+ ++p;
+
+ *data = reinterpret_cast<const char*>(p);
+}
+
+/**
+ * Varint encode a 64 bit integer.
+ *
+ * @tparam T An output iterator type.
+ * @param data Output iterator the varint encoded value will be written to
+ * byte by byte.
+ * @param value The integer that will be encoded.
+ * @returns the number of bytes written
+ * @throws Any exception thrown by increment or dereference operator on data.
+ * @deprecated Use add_varint_to_buffer() instead.
+ */
+template <typename T>
+inline int write_varint(T data, uint64_t value) {
+ int n = 1;
+
+ while (value >= 0x80U) {
+ *data++ = static_cast<char>((value & 0x7fU) | 0x80U);
+ value >>= 7U;
+ ++n;
+ }
+ *data = static_cast<char>(value);
+
+ return n;
+}
+
+/**
+ * Varint encode a 64 bit integer.
+ *
+ * @tparam TBuffer A buffer type.
+ * @param buffer Output buffer the varint will be written to.
+ * @param value The integer that will be encoded.
+ * @throws Any exception thrown by calling the buffer_push_back() function.
+ */
+template <typename TBuffer>
+inline void add_varint_to_buffer(TBuffer* buffer, uint64_t value) {
+ while (value >= 0x80U) {
+ buffer_customization<TBuffer>::push_back(buffer, static_cast<char>((value & 0x7fU) | 0x80U));
+ value >>= 7U;
+ }
+ buffer_customization<TBuffer>::push_back(buffer, static_cast<char>(value));
+}
+
+/**
+ * Varint encode a 64 bit integer.
+ *
+ * @param data Where to add the varint. There must be enough space available!
+ * @param value The integer that will be encoded.
+ * @returns the number of bytes written
+ */
+inline int add_varint_to_buffer(char* data, uint64_t value) noexcept {
+ int n = 1;
+
+ while (value >= 0x80U) {
+ *data++ = static_cast<char>((value & 0x7fU) | 0x80U);
+ value >>= 7U;
+ ++n;
+ }
+ *data = static_cast<char>(value);
+
+ return n;
+}
+
+/**
+ * Get the length of the varint the specified value would produce.
+ *
+ * @param value The integer to be encoded.
+ * @returns the number of bytes the varint would have if we created it.
+ */
+inline int length_of_varint(uint64_t value) noexcept {
+ int n = 1;
+
+ while (value >= 0x80U) {
+ value >>= 7U;
+ ++n;
+ }
+
+ return n;
+}
+
+/**
+ * ZigZag encodes a 32 bit integer.
+ */
+constexpr uint32_t encode_zigzag32(int32_t value) noexcept {
+ return (static_cast<uint32_t>(value) << 1U) ^ static_cast<uint32_t>(-static_cast<int32_t>(static_cast<uint32_t>(value) >> 31U));
+}
+
+/**
+ * ZigZag encodes a 64 bit integer.
+ */
+constexpr uint64_t encode_zigzag64(int64_t value) noexcept {
+ return (static_cast<uint64_t>(value) << 1U) ^ static_cast<uint64_t>(-static_cast<int64_t>(static_cast<uint64_t>(value) >> 63U));
+}
+
+/**
+ * Decodes a 32 bit ZigZag-encoded integer.
+ */
+constexpr int32_t decode_zigzag32(uint32_t value) noexcept {
+ return static_cast<int32_t>((value >> 1U) ^ static_cast<uint32_t>(-static_cast<int32_t>(value & 1U)));
+}
+
+/**
+ * Decodes a 64 bit ZigZag-encoded integer.
+ */
+constexpr int64_t decode_zigzag64(uint64_t value) noexcept {
+ return static_cast<int64_t>((value >> 1U) ^ static_cast<uint64_t>(-static_cast<int64_t>(value & 1U)));
+}
+
+} // end namespace protozero
+
+#endif // PROTOZERO_VARINT_HPP
diff --git a/thirdparty/protozero/include/protozero/version.hpp b/thirdparty/protozero/include/protozero/version.hpp
new file mode 100644
index 000000000..760870f97
--- /dev/null
+++ b/thirdparty/protozero/include/protozero/version.hpp
@@ -0,0 +1,34 @@
+#ifndef PROTOZERO_VERSION_HPP
+#define PROTOZERO_VERSION_HPP
+
+/*****************************************************************************
+
+protozero - Minimalistic protocol buffer decoder and encoder in C++.
+
+This file is from https://github.com/mapbox/protozero where you can find more
+documentation.
+
+*****************************************************************************/
+
+/**
+ * @file version.hpp
+ *
+ * @brief Contains macros defining the protozero version.
+ */
+
+/// The major version number
+#define PROTOZERO_VERSION_MAJOR 1
+
+/// The minor version number
+#define PROTOZERO_VERSION_MINOR 8
+
+/// The patch number
+#define PROTOZERO_VERSION_PATCH 1
+
+/// The complete version number
+#define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH)
+
+/// Version number as string
+#define PROTOZERO_VERSION_STRING "1.8.1"
+
+#endif // PROTOZERO_VERSION_HPP
diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua
index d745a5109..c9acfa018 100644
--- a/thirdparty/xmake.lua
+++ b/thirdparty/xmake.lua
@@ -43,3 +43,9 @@ target('rpmalloc')
add_files("rpmalloc/rpmalloc.c")
add_headerfiles("rpmalloc/**.h")
add_includedirs("rpmalloc", {public=true})
+
+target('protozero')
+ set_kind('headeronly')
+ set_group('thirdparty')
+ add_headerfiles("protozero/**.hpp")
+ add_includedirs("protozero/include", {public=true})
diff --git a/xmake.lua b/xmake.lua
index a62cbfc9e..6e4e0f557 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -97,7 +97,8 @@ if is_os("windows") then
"NOMINMAX", -- stop Windows SDK defining 'min' and 'max'
"NOGDI", -- otherwise Windows.h defines 'GetObject'
"WIN32_LEAN_AND_MEAN", -- cut down Windows.h
- "_WIN32_WINNT=0x0A00"
+ "_WIN32_WINNT=0x0A00",
+ "_WINSOCK_DEPRECATED_NO_WARNINGS" -- let us use the ANSI functions
)
-- Make builds more deterministic and portable
add_cxxflags("/d1trimfile:$(curdir)\\") -- eliminates the base path from __FILE__ paths
@@ -202,14 +203,15 @@ set_symbols("debug")
includes("thirdparty")
includes("src/transports")
includes("src/zenbase")
-includes("src/zencore", "src/zencore-test")
-includes("src/zenhttp", "src/zenhttp-test")
-includes("src/zennet", "src/zennet-test")
-includes("src/zenremotestore", "src/zenremotestore-test")
-includes("src/zenstore", "src/zenstore-test")
-includes("src/zenutil", "src/zenutil-test")
+includes("src/zencore", "src/zencore-test")
+includes("src/zenhttp", "src/zenhttp-test")
+includes("src/zennet", "src/zennet-test")
+includes("src/zenremotestore", "src/zenremotestore-test")
+includes("src/zenstore", "src/zenstore-test")
+includes("src/zentelemetry", "src/zentelemetry-test")
+includes("src/zenutil", "src/zenutil-test")
includes("src/zenvfs")
-includes("src/zenserver", "src/zenserver-test")
+includes("src/zenserver", "src/zenserver-test")
includes("src/zen")
includes("src/zentest-appstub")