add otel instrumentation (#581)

this change adds OTEL tracing to a few places

* Top-level application lifecycle (config/init/cleanup, main loop)
* http.sys requests

it also brings some otlptrace optimizations and dynamic configuration of tracing. OTLP tracing is currently always disabled

5 files changed, 141 insertions, 21 deletions

diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index e529fb76e..085275195 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -26,6 +26,7 @@
 #include <zencore/testing.h>
 #include <zencore/thread.h>
 #include <zenhttp/packageformat.h>
+#include <zentelemetry/otlptrace.h>
 
 #include <charconv>
 #include <mutex>
@@ -462,7 +463,7 @@ HttpService::HandlePackageRequest(HttpServerRequest& HttpServiceRequest)
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpServerRequest::HttpServerRequest()
+HttpServerRequest::HttpServerRequest(HttpService& Service) : m_BaseUri(Service.BaseUri())
 {
 }
 
@@ -896,7 +897,7 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request)
 
 	// First try new-style matcher routes
 
-	for (const auto& Handler : m_MatcherEndpoints)
+	for (const MatcherEndpoint& Handler : m_MatcherEndpoints)
 	{
 		if ((Handler.Verbs & Verb) == Verb)
 		{
@@ -965,6 +966,16 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request)
 
 			if (IsMatch && UriPos == UriLen)
 			{
+#if ZEN_WITH_OTEL
+				if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan())
+				{
+					ExtendableStringBuilder<128> RoutePath;
+					RoutePath.Append(Request.BaseUri());
+					RoutePath.Append(Handler.Pattern);
+					ActiveSpan->AddAttribute("http.route"sv, RoutePath.ToView());
+				}
+#endif
+
 				RouterRequest.m_CapturedSegments = std::move(CapturedSegments);
 				Handler.Handler(RouterRequest);
 
@@ -979,6 +990,16 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request)
 	{
 		if ((Handler.Verbs & Verb) == Verb && regex_match(begin(Uri), end(Uri), RouterRequest.m_Match, Handler.RegEx))
 		{
+#if ZEN_WITH_OTEL
+			if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan())
+			{
+				ExtendableStringBuilder<128> RoutePath;
+				RoutePath.Append(Request.BaseUri());
+				RoutePath.Append(Handler.Pattern);
+				ActiveSpan->AddAttribute("http.route"sv, RoutePath.ToView());
+			}
+#endif
+
 			Handler.Handler(RouterRequest);
 
 			return true;  // Route matched
@@ -1277,9 +1298,17 @@ TEST_CASE("http.common")
 {
 	using namespace std::literals;
 
+	struct TestHttpService : public HttpService
+	{
+		TestHttpService() = default;
+
+		virtual const char* BaseUri() const override { return "/test"; }
+		virtual void		HandleRequest(HttpServerRequest& HttpServiceRequest) override { ZEN_UNUSED(HttpServiceRequest); }
+	};
+
 	struct TestHttpServerRequest : public HttpServerRequest
 	{
-		TestHttpServerRequest(std::string_view Uri) { m_Uri = Uri; }
+		TestHttpServerRequest(HttpService& Service, std::string_view Uri) : HttpServerRequest(Service) { m_Uri = Uri; }
 		virtual IoBuffer ReadPayload() override { return IoBuffer(); }
 		virtual void	 WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) override
 		{
@@ -1308,6 +1337,8 @@ TEST_CASE("http.common")
 			HandledA = HandledAA = false;
 		};
 
+		TestHttpService Service;
+
 		HttpRequestRouter r;
 		r.AddPattern("a", "([[:alpha:]]+)");
 		r.RegisterRoute(
@@ -1328,7 +1359,7 @@ TEST_CASE("http.common")
 
 		{
 			Reset();
-			TestHttpServerRequest req{"abc"sv};
+			TestHttpServerRequest req(Service, "abc"sv);
 			r.HandleRequest(req);
 			CHECK(HandledA);
 			CHECK(!HandledAA);
@@ -1338,7 +1369,7 @@ TEST_CASE("http.common")
 
 		{
 			Reset();
-			TestHttpServerRequest req{"abc/def"sv};
+			TestHttpServerRequest req{Service, "abc/def"sv};
 			r.HandleRequest(req);
 			CHECK(!HandledA);
 			CHECK(HandledAA);
@@ -1349,14 +1380,14 @@ TEST_CASE("http.common")
 
 		{
 			Reset();
-			TestHttpServerRequest req{"123"sv};
+			TestHttpServerRequest req{Service, "123"sv};
 			r.HandleRequest(req);
 			CHECK(!HandledA);
 		}
 
 		{
 			Reset();
-			TestHttpServerRequest req{"a123"sv};
+			TestHttpServerRequest req{Service, "a123"sv};
 			r.HandleRequest(req);
 			CHECK(!HandledA);
 		}
@@ -1374,6 +1405,7 @@ TEST_CASE("http.common")
 			Captures.clear();
 		};
 
+		TestHttpService	  Service;
 		HttpRequestRouter r;
 		r.AddMatcher("a", [](std::string_view In) -> bool { return In.length() % 2 == 0; });
 		r.AddMatcher("b", [](std::string_view In) -> bool { return In.length() % 3 == 0; });
@@ -1408,7 +1440,7 @@ TEST_CASE("http.common")
 
 		{
 			Reset();
-			TestHttpServerRequest req{"ab"sv};
+			TestHttpServerRequest req{Service, "ab"sv};
 			r.HandleRequest(req);
 			CHECK(HandledA);
 			CHECK(!HandledAA);
@@ -1420,7 +1452,7 @@ TEST_CASE("http.common")
 
 		{
 			Reset();
-			TestHttpServerRequest req{"ab/def"sv};
+			TestHttpServerRequest req{Service, "ab/def"sv};
 			r.HandleRequest(req);
 			CHECK(!HandledA);
 			CHECK(!HandledAA);
@@ -1432,7 +1464,7 @@ TEST_CASE("http.common")
 
 		{
 			Reset();
-			TestHttpServerRequest req{"ab/and/def"sv};
+			TestHttpServerRequest req{Service, "ab/and/def"sv};
 			r.HandleRequest(req);
 			CHECK(!HandledA);
 			CHECK(!HandledAA);
@@ -1445,7 +1477,7 @@ TEST_CASE("http.common")
 
 		{
 			Reset();
-			TestHttpServerRequest req{"123"sv};
+			TestHttpServerRequest req{Service, "123"sv};
 			r.HandleRequest(req);
 			CHECK(!HandledA);
 			CHECK(!HandledAA);
@@ -1454,7 +1486,7 @@ TEST_CASE("http.common")
 
 		{
 			Reset();
-			TestHttpServerRequest req{"a123"sv};
+			TestHttpServerRequest req{Service, "a123"sv};
 			r.HandleRequest(req);
 			CHECK(HandledA);
 			CHECK(!HandledAA);
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
index 9e5c41ab7..074e40734 100644
--- a/src/zenhttp/include/zenhttp/httpserver.h
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -24,13 +24,14 @@
 namespace zen {
 
 class CbPackage;
+class HttpService;
 
 /** HTTP Server Request
  */
 class HttpServerRequest
 {
 public:
-	HttpServerRequest();
+	explicit HttpServerRequest(HttpService& Service);
 	~HttpServerRequest();
 
 	// Synchronous operations
@@ -38,6 +39,7 @@ public:
 	[[nodiscard]] inline std::string_view RelativeUri() const { return m_Uri; }	 // Returns URI without service prefix
 	[[nodiscard]] std::string_view		  RelativeUriWithExtension() const { return m_UriWithExtension; }
 	[[nodiscard]] inline std::string_view QueryString() const { return m_QueryString; }
+	[[nodiscard]] inline std::string_view BaseUri() const { return m_BaseUri; }	 // Service prefix
 
 	struct QueryParams
 	{
@@ -120,7 +122,8 @@ protected:
 	HttpContentType	 m_ContentType	 = HttpContentType::kBinary;
 	HttpContentType	 m_AcceptType	 = HttpContentType::kUnknownContentType;
 	uint64_t		 m_ContentLength = ~0ull;
-	std::string_view m_Uri;
+	std::string_view m_BaseUri;	 // Base URI path of the service handling this request
+	std::string_view m_Uri;		 // URI without service prefix
 	std::string_view m_UriWithExtension;
 	std::string_view m_QueryString;
 	mutable uint32_t m_RequestId = ~uint32_t(0);
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index 12aee1ee0..4a0edfd29 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -1017,7 +1017,8 @@ private:
 //////////////////////////////////////////////////////////////////////////
 
 HttpAsioServerRequest::HttpAsioServerRequest(HttpRequestParser& Request, HttpService& Service, IoBuffer PayloadBuffer)
-: m_Request(Request)
+: HttpServerRequest(Service)
+, m_Request(Request)
 , m_PayloadBuffer(std::move(PayloadBuffer))
 {
 	const int PrefixLength = Service.UriPrefixLength();
diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp
index 426e62179..44a5b71d0 100644
--- a/src/zenhttp/servers/httpplugin.cpp
+++ b/src/zenhttp/servers/httpplugin.cpp
@@ -552,7 +552,8 @@ HttpPluginConnectionHandler::TerminateConnection()
 //////////////////////////////////////////////////////////////////////////
 
 HttpPluginServerRequest::HttpPluginServerRequest(HttpRequestParser& Request, HttpService& Service, IoBuffer PayloadBuffer)
-: m_Request(Request)
+: HttpServerRequest(Service)
+, m_Request(Request)
 , m_PayloadBuffer(std::move(PayloadBuffer))
 {
 	ZEN_MEMSCOPE(GetHttppluginTag());
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index 7b02f95f1..87b66dcd1 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -15,6 +15,7 @@
 #include <zencore/timer.h>
 #include <zencore/trace.h>
 #include <zenhttp/packageformat.h>
+#include <zentelemetry/otlptrace.h>
 
 #include <EASTL/fixed_vector.h>
 
@@ -36,6 +37,41 @@ GetHttpsysTag()
 	return HttpsysTag;
 }
 
+static void
+GetAddressString(StringBuilderBase& OutString, const SOCKADDR* SockAddr, bool IncludePort = true)
+{
+	if (SockAddr)
+	{
+		// When a port is desired, use WSAAddressToStringA which includes the port.
+		if (IncludePort)
+		{
+			CHAR		AddrBuf[64] = {};
+			DWORD		AddrBufLen	= sizeof(AddrBuf);
+			const DWORD AddrLen		= (SockAddr->sa_family == AF_INET) ? sizeof(SOCKADDR_IN) : sizeof(SOCKADDR_IN6);
+
+			if (WSAAddressToStringA((LPSOCKADDR)SockAddr, AddrLen, nullptr, AddrBuf, &AddrBufLen) == 0)
+			{
+				OutString.Append(AddrBuf, AddrBufLen);
+				return;
+			}
+		}
+		else
+		{
+			// When port should be omitted, use getnameinfo with NI_NUMERICHOST to get only the numeric address.
+			CHAR	  HostBuf[64] = {};
+			const int SockLen	  = (SockAddr->sa_family == AF_INET) ? sizeof(SOCKADDR_IN) : sizeof(SOCKADDR_IN6);
+
+			if (getnameinfo((const SOCKADDR*)SockAddr, SockLen, HostBuf, (DWORD)sizeof(HostBuf), nullptr, 0, NI_NUMERICHOST) == 0)
+			{
+				OutString.Append(HostBuf, (DWORD)strlen(HostBuf));
+				return;
+			}
+		}
+	}
+
+	OutString.Append("unknown");
+}
+
 /**
  * @brief Windows implementation of HTTP server based on http.sys
  *
@@ -382,6 +418,8 @@ public:
 	virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) override;
 	void						   SuppressResponseBody();	// typically used for HEAD requests
 
+	inline int64_t GetResponseBodySize() const { return m_TotalDataSize; }
+
 private:
 	eastl::fixed_vector<HTTP_DATA_CHUNK, 16> m_HttpDataChunks;
 	uint64_t								 m_TotalDataSize	   = 0;	 // Sum of all chunk sizes
@@ -1605,6 +1643,27 @@ HttpSysTransaction::InvokeRequestHandler(HttpService& Service, IoBuffer Payload)
 
 	// Default request handling
 
+#	if ZEN_WITH_OTEL
+	std::string_view Verb = ToString(ThisRequest.RequestVerb());
+	std::string_view Uri  = ThisRequest.m_UriUtf8.ToView();
+
+	ExtendableStringBuilder<64> SpanName;
+	SpanName << Verb << " " << Uri;
+	otel::ScopedSpan HttpSpan(SpanName.ToView(), [&](otel::Span& Span) {
+		Span.AddAttribute("http.request.method"sv, Verb);
+		Span.AddAttribute("url.path"sv, Uri);
+		// FIXME: should be total size including headers etc according to spec
+		Span.AddAttribute("http.request.size"sv, static_cast<int64_t>(ThisRequest.ContentLength()));
+		Span.SetKind(otel::Span::Kind::kServer);
+
+		// client.address
+		const SOCKADDR*				SockAddr = ThisRequest.m_HttpTx.HttpRequest()->Address.pRemoteAddress;
+		ExtendableStringBuilder<64> ClientAddr;
+		GetAddressString(ClientAddr, SockAddr, /* IncludePort */ false);
+		Span.AddAttribute("client.address"sv, ClientAddr.ToView());
+	});
+#	endif
+
 	if (!HandlePackageOffers(Service, ThisRequest, m_PackageHandler))
 	{
 		Service.HandleRequest(ThisRequest);
@@ -1616,7 +1675,8 @@ HttpSysTransaction::InvokeRequestHandler(HttpService& Service, IoBuffer Payload)
 //////////////////////////////////////////////////////////////////////////
 
 HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& Service, IoBuffer PayloadBuffer)
-: m_HttpTx(Tx)
+: HttpServerRequest(Service)
+, m_HttpTx(Tx)
 , m_PayloadBuffer(std::move(PayloadBuffer))
 {
 	const HTTP_REQUEST* HttpRequestPtr = Tx.HttpRequest();
@@ -1626,16 +1686,16 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService&
 
 	HttpContentType AcceptContentType = HttpContentType::kUnknownContentType;
 
+	WideToUtf8({(wchar_t*)HttpRequestPtr->CookedUrl.pAbsPath, gsl::narrow<size_t>(AbsPathLength)}, m_UriUtf8);
+
 	if (AbsPathLength >= PrefixLength)
 	{
 		// We convert the URI immediately because most of the code involved prefers to deal
 		// with utf8. This is overhead which I'd prefer to avoid but for now we just have
 		// to live with it
 
-		WideToUtf8({(wchar_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)},
-				   m_UriUtf8);
-
 		std::string_view UriSuffix8{m_UriUtf8};
+		UriSuffix8.remove_prefix(PrefixLength);
 
 		m_UriWithExtension = UriSuffix8;  // Retain URI with extension for user access
 		m_Uri			   = UriSuffix8;
@@ -1662,7 +1722,6 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService&
 	}
 	else
 	{
-		m_UriUtf8.Reset();
 		m_Uri			   = {};
 		m_UriWithExtension = {};
 	}
@@ -1771,6 +1830,14 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode)
 
 	m_NextCompletionHandler = Response;
 
+#	if ZEN_WITH_OTEL
+	if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan())
+	{
+		ActiveSpan->AddAttribute("http.response.body.size"sv, Response->GetResponseBodySize());
+		ActiveSpan->AddAttribute("http.response.status_code"sv, static_cast<int64_t>(ResponseCode));
+	}
+#	endif
+
 	SetIsHandled();
 }
 
@@ -1790,6 +1857,14 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentTy
 
 	m_NextCompletionHandler = Response;
 
+#	if ZEN_WITH_OTEL
+	if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan())
+	{
+		ActiveSpan->AddAttribute("http.response.body.size"sv, Response->GetResponseBodySize());
+		ActiveSpan->AddAttribute("http.response.status_code"sv, static_cast<int64_t>(ResponseCode));
+	}
+#	endif
+
 	SetIsHandled();
 }
 
@@ -1810,6 +1885,14 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentTy
 
 	m_NextCompletionHandler = Response;
 
+#	if ZEN_WITH_OTEL
+	if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan())
+	{
+		ActiveSpan->AddAttribute("http.response.body.size"sv, Response->GetResponseBodySize());
+		ActiveSpan->AddAttribute("http.response.status_code"sv, static_cast<int64_t>(ResponseCode));
+	}
+#	endif
+
 	SetIsHandled();
 }