add support for OTLP logging/tracing (#599)

- adds `zentelemetry` project which houses new functionality for serializing logs and traces in OpenTelemetry Protocol format (OTLP)
- moved existing stats functionality from `zencore` to `zentelemetry`
- adds `TRefCounted<T>` for vtable-less refcounting
- adds `MemoryArena` class which allows for linear allocation of memory from chunks
- adds `protozero` which is used to encode OTLP protobuf messages

10 files changed, 168 insertions, 6 deletions

diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp
index 50cf62274..90af79651 100644
--- a/src/zenserver/diag/logging.cpp
+++ b/src/zenserver/diag/logging.cpp
@@ -12,6 +12,8 @@
 #include <zenutil/logging.h>
 #include <zenutil/logging/rotatingfilesink.h>
 
+#include "otlphttp.h"
+
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <spdlog/spdlog.h>
 ZEN_THIRD_PARTY_INCLUDES_END
@@ -73,6 +75,16 @@ InitializeServerLogging(const ZenServerConfig& InOptions)
 	spdlog::apply_logger_env_levels(ZenClientLogger);
 	spdlog::register_logger(ZenClientLogger);
 
+	//
+
+#if ZEN_WITH_OTEL
+	if (false)
+	{
+		auto OtelSink = std::make_shared<zen::logging::OtelHttpProtobufSink>("http://signoz.localdomain:4318");
+		zen::logging::Default().SpdLogger->sinks().push_back(std::move(OtelSink));
+	}
+#endif
+
 	FinishInitializeLogging(LogOptions);
 
 	const zen::Oid ServerSessionId = zen::GetSessionId();
diff --git a/src/zenserver/diag/otlphttp.cpp b/src/zenserver/diag/otlphttp.cpp
new file mode 100644
index 000000000..d62ccccb6
--- /dev/null
+++ b/src/zenserver/diag/otlphttp.cpp
@@ -0,0 +1,83 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "otlphttp.h"
+
+#include <zencore/config.h>
+#include <zencore/process.h>
+#include <zencore/session.h>
+#include <zencore/system.h>
+#include <zentelemetry/otlpencoder.h>
+#include <protozero/buffer_string.hpp>
+#include <protozero/pbf_builder.hpp>
+
+#if ZEN_WITH_OTEL
+
+namespace zen::logging {
+
+//////////////////////////////////////////////////////////////////////////
+
+OtelHttpProtobufSink::OtelHttpProtobufSink(const std::string_view& Uri) : m_OtelHttp(Uri)
+{
+	m_Encoder.AddResourceAttribute("service.name", "zenserver");
+	m_Encoder.AddResourceAttribute("service.instance.id", GetSessionIdString());
+	m_Encoder.AddResourceAttribute("service.namespace", "zen");
+	m_Encoder.AddResourceAttribute("service.version", ZEN_CFG_VERSION);
+	m_Encoder.AddResourceAttribute("host.name", GetMachineName());
+	m_Encoder.AddResourceAttribute("session.id", GetSessionIdString());
+	m_Encoder.AddResourceAttribute("process.id", zen::GetCurrentProcessId());
+
+	m_TraceRecorder = new TraceRecorder(this);
+	otel::SetTraceRecorder(m_TraceRecorder);
+}
+
+OtelHttpProtobufSink::~OtelHttpProtobufSink()
+{
+	otel::SetTraceRecorder({});
+}
+
+void
+OtelHttpProtobufSink::RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans)
+{
+	std::string Data = m_Encoder.FormatOtelTrace(Trace, Spans);
+
+	IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
+	Payload.SetContentType(ZenContentType::kProtobuf);
+
+	auto Result = m_OtelHttp.Post("/v1/traces", Payload);
+}
+
+void
+OtelHttpProtobufSink::TraceRecorder::RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans)
+{
+	m_Sink->RecordSpans(Trace, Spans);
+}
+
+void
+OtelHttpProtobufSink::log(const spdlog::details::log_msg& Msg)
+{
+	{
+		std::string Data = m_Encoder.FormatOtelProtobuf(Msg);
+
+		IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
+		Payload.SetContentType(ZenContentType::kProtobuf);
+
+		auto Result = m_OtelHttp.Post("/v1/logs", Payload);
+	}
+
+	{
+		std::string Data = m_Encoder.FormatOtelMetrics();
+
+		IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
+		Payload.SetContentType(ZenContentType::kProtobuf);
+
+		auto Result = m_OtelHttp.Post("/v1/metrics", Payload);
+	}
+}
+void
+OtelHttpProtobufSink::flush()
+{
+}
+
+}  // namespace zen::logging
+
+#endif
diff --git a/src/zenserver/diag/otlphttp.h b/src/zenserver/diag/otlphttp.h
new file mode 100644
index 000000000..2281bdcc0
--- /dev/null
+++ b/src/zenserver/diag/otlphttp.h
@@ -0,0 +1,64 @@
+
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <spdlog/sinks/sink.h>
+#include <zencore/zencore.h>
+#include <zenhttp/httpclient.h>
+#include <zentelemetry/otlpencoder.h>
+#include <zentelemetry/otlptrace.h>
+
+#if ZEN_WITH_OTEL
+
+namespace zen::logging {
+
+/**
+ * OTLP/HTTP sink for spdlog
+ *
+ * Sends log messages and traces to an OpenTelemetry collector via OTLP over HTTP
+ */
+
+class OtelHttpProtobufSink : public spdlog::sinks::sink
+{
+public:
+	// Note that this URI should be the base URI of the OTLP HTTP endpoint, e.g.
+	// "http://otel-collector:4318"
+	OtelHttpProtobufSink(const std::string_view& Uri);
+	~OtelHttpProtobufSink();
+
+	OtelHttpProtobufSink(const OtelHttpProtobufSink&) = delete;
+	OtelHttpProtobufSink& operator=(const OtelHttpProtobufSink&) = delete;
+
+private:
+	virtual void log(const spdlog::details::log_msg& Msg) override;
+	virtual void flush() override;
+	virtual void set_pattern(const std::string& pattern) override { ZEN_UNUSED(pattern); }
+	virtual void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) override { ZEN_UNUSED(sink_formatter); }
+
+	void RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans);
+
+	// This is just a thin wrapper to call back into the sink while participating in
+	// reference counting from the OTEL trace back-end
+	class TraceRecorder : public zen::otel::TraceRecorder
+	{
+	public:
+		TraceRecorder(OtelHttpProtobufSink* InSink) : m_Sink(InSink) {}
+
+	private:
+		TraceRecorder(const TraceRecorder&) = delete;
+		TraceRecorder& operator=(const TraceRecorder&) = delete;
+
+		virtual void RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans) override;
+
+		OtelHttpProtobufSink* m_Sink;
+	};
+
+	HttpClient		   m_OtelHttp;
+	OtlpEncoder		   m_Encoder;
+	Ref<TraceRecorder> m_TraceRecorder;
+};
+
+}  // namespace zen::logging
+
+#endif
\ No newline at end of file
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h
index 50cb5db12..e10986411 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.h
+++ b/src/zenserver/storage/buildstore/httpbuildstore.h
@@ -2,10 +2,10 @@
 
 #pragma once
 
-#include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
 #include <zenhttp/httpstatus.h>
+#include <zentelemetry/stats.h>
 
 #include <filesystem>
 
diff --git a/src/zenserver/storage/cache/httpstructuredcache.h b/src/zenserver/storage/cache/httpstructuredcache.h
index a157148c9..5a795c215 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.h
+++ b/src/zenserver/storage/cache/httpstructuredcache.h
@@ -2,12 +2,12 @@
 
 #pragma once
 
-#include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
 #include <zenhttp/httpstatus.h>
 #include <zenstore/cache/cache.h>
 #include <zenstore/cache/cacherpc.h>
+#include <zentelemetry/stats.h>
 #include <zenutil/openprocesscache.h>
 
 #include <memory>
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h
index f0a0bcfa1..f6fe63614 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.h
+++ b/src/zenserver/storage/projectstore/httpprojectstore.h
@@ -2,11 +2,11 @@
 
 #pragma once
 
-#include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
 #include <zenhttp/httpstatus.h>
 #include <zenstore/cidstore.h>
+#include <zentelemetry/stats.h>
 
 namespace zen {
 
diff --git a/src/zenserver/storage/upstream/upstreamcache.cpp b/src/zenserver/storage/upstream/upstreamcache.cpp
index f7ae5f973..6c489c5d3 100644
--- a/src/zenserver/storage/upstream/upstreamcache.cpp
+++ b/src/zenserver/storage/upstream/upstreamcache.cpp
@@ -9,10 +9,10 @@
 #include <zencore/compactbinarypackage.h>
 #include <zencore/compactbinaryvalidation.h>
 #include <zencore/fmtutils.h>
-#include <zencore/stats.h>
 #include <zencore/stream.h>
 #include <zencore/timer.h>
 #include <zencore/trace.h>
+#include <zentelemetry/stats.h>
 
 #include <zenhttp/httpclientauth.h>
 #include <zenhttp/packageformat.h>
diff --git a/src/zenserver/storage/upstream/upstreamcache.h b/src/zenserver/storage/upstream/upstreamcache.h
index d5d61c8d9..c0c8a7ff9 100644
--- a/src/zenserver/storage/upstream/upstreamcache.h
+++ b/src/zenserver/storage/upstream/upstreamcache.h
@@ -6,10 +6,10 @@
 #include <zencore/compress.h>
 #include <zencore/iobuffer.h>
 #include <zencore/iohash.h>
-#include <zencore/stats.h>
 #include <zencore/zencore.h>
 #include <zenstore/cache/cache.h>
 #include <zenstore/cache/upstreamcacheclient.h>
+#include <zentelemetry/stats.h>
 
 #include <atomic>
 #include <chrono>
diff --git a/src/zenserver/storage/workspaces/httpworkspaces.h b/src/zenserver/storage/workspaces/httpworkspaces.h
index 89a8e8bdc..888a34b4d 100644
--- a/src/zenserver/storage/workspaces/httpworkspaces.h
+++ b/src/zenserver/storage/workspaces/httpworkspaces.h
@@ -2,10 +2,10 @@
 
 #pragma once
 
-#include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
 #include <zenhttp/httpstatus.h>
+#include <zentelemetry/stats.h>
 
 namespace zen {
 
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index 57105045d..483bfd5aa 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -7,6 +7,7 @@ target("zenserver")
              "zennet",
              "zenremotestore",
              "zenstore",
+             "zentelemetry",
              "zenutil",
              "zenvfs")
     add_headerfiles("**.h")
@@ -17,6 +18,8 @@ target("zenserver")
     add_includedirs(".")
     set_symbols("debug")
 
+    add_deps("protozero")
+
     if is_mode("release") then
         set_optimize("fastest")
     end