add otel instrumentation (#581)

this change adds OTEL tracing to a few places

* Top-level application lifecycle (config/init/cleanup, main loop)
* http.sys requests

it also brings some otlptrace optimizations and dynamic configuration of tracing. OTLP tracing is currently always disabled

1 files changed, 12 insertions, 0 deletions

diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index 827f22ecc..93a01d9d7 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -28,6 +28,7 @@
 #include <zenstore/scrubcontext.h>
 #include <zenstore/vfsimpl.h>
 #include <zenstore/workspaces.h>
+#include <zentelemetry/otlptrace.h>
 #include <zenutil/service.h>
 #include <zenutil/workerpools.h>
 #include <zenutil/zenserverprocess.h>
@@ -185,6 +186,8 @@ ZenStorageServer::RegisterServices()
 void
 ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions)
 {
+	ZEN_OTEL_SPAN("InitializeServices");
+
 	InitializeAuthentication(ServerOptions);
 
 	ZEN_INFO("initializing storage");
@@ -206,6 +209,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
 
 	if (ServerOptions.WorksSpacesConfig.Enabled)
 	{
+		ZEN_OTEL_SPAN("InitializeWorkspaces");
+
 		m_Workspaces.reset(new Workspaces());
 		m_HttpWorkspacesService.reset(
 			new HttpWorkspacesService(m_StatusService,
@@ -239,6 +244,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
 
 	if (ServerOptions.ObjectStoreEnabled)
 	{
+		ZEN_OTEL_SPAN("InitializeObjectStore");
+
 		ObjectStoreConfig ObjCfg;
 		ObjCfg.RootDirectory = m_DataRoot / "obj";
 
@@ -254,6 +261,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
 
 	if (ServerOptions.BuildStoreConfig.Enabled)
 	{
+		ZEN_OTEL_SPAN("InitializeBuildStore");
+
 		m_BuildStoreService = std::make_unique<HttpBuildStoreService>(m_StatusService, m_StatsService, *m_BuildStore);
 	}
 
@@ -347,6 +356,7 @@ ZenStorageServer::InitializeAuthentication(const ZenStorageServerConfig& ServerO
 void
 ZenStorageServer::InitializeState(const ZenStorageServerConfig& ServerOptions)
 {
+	ZEN_OTEL_SPAN("InitializeState");
 	ZEN_TRACE_CPU("ZenStorageServer::InitializeState");
 
 	// Check root manifest to deal with schema versioning
@@ -451,6 +461,7 @@ ZenStorageServer::InitializeState(const ZenStorageServerConfig& ServerOptions)
 
 	if (WipeState)
 	{
+		ZEN_OTEL_SPAN("WipingState");
 		ZEN_WARN("Wiping state at '{}' - reason: '{}'", m_DataRoot, WipeReason);
 
 		std::error_code Ec;
@@ -517,6 +528,7 @@ ZenStorageServer::InitializeState(const ZenStorageServerConfig& ServerOptions)
 void
 ZenStorageServer::InitializeStructuredCache(const ZenStorageServerConfig& ServerOptions)
 {
+	ZEN_OTEL_SPAN("InitializeStructuredCache");
 	ZEN_TRACE_CPU("ZenStorageServer::InitializeStructuredCache");
 
 	using namespace std::literals;