HTTP request logging (#6)

this change adds support for tracing http payloads when using the asio path. This was already supported when using the `--http=plugin` path and this change moves some code into a shared class for reuse.

4 files changed, 108 insertions, 31 deletions

diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index 7ef0437c3..de71eb0a7 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -1,6 +1,7 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #include "httpasio.h"
+#include "httptracer.h"
 
 #include <zencore/except.h>
 #include <zencore/logging.h>
@@ -62,6 +63,7 @@ public:
 	HttpAsioServerImpl();
 	~HttpAsioServerImpl();
 
+	void		 Initialize(std::filesystem::path DataDir);
 	int			 Start(uint16_t Port, bool ForceLooopback, int ThreadCount);
 	void		 Stop();
 	void		 RegisterService(const char* UrlPath, HttpService& Service);
@@ -72,6 +74,9 @@ public:
 	std::unique_ptr<asio_http::HttpAcceptor> m_Acceptor;
 	std::vector<std::thread>				 m_ThreadPool;
 
+	LoggerRef		 m_RequestLog;
+	HttpServerTracer m_RequestTracer;
+
 	struct ServiceEntry
 	{
 		std::string	 ServiceUrlPath;
@@ -441,9 +446,29 @@ HttpServerConnection::HandleRequest()
 	{
 		ZEN_TRACE_CPU("asio::HandleRequest");
 
+		const uint32_t RequestNumber = m_RequestCounter.load(std::memory_order_relaxed);
+
 		HttpAsioServerRequest Request(m_RequestData, *Service, m_RequestData.Body());
 
-		ZEN_TRACE_VERBOSE("handle request, connection: {}, request: {}'", m_ConnectionId, m_RequestCounter.load(std::memory_order_relaxed));
+		ZEN_TRACE_VERBOSE("handle request, connection: {}, request: {}'", m_ConnectionId, RequestNumber);
+
+		const HttpVerb		   RequestVerb = Request.RequestVerb();
+		const std::string_view Uri		   = Request.RelativeUri();
+
+		if (m_Server.m_RequestLog.ShouldLog(logging::level::Trace))
+		{
+			ZEN_LOG_TRACE(m_Server.m_RequestLog,
+						  "connection #{} Handling Request: {} {} ({} bytes ({}), accept: {})",
+						  m_ConnectionId,
+						  ToString(RequestVerb),
+						  Uri,
+						  Request.ContentLength(),
+						  ToString(Request.RequestContentType()),
+						  ToString(Request.AcceptContentType()));
+
+			m_Server.m_RequestTracer.WriteDebugPayload(fmt::format("request_{}_{}.bin", m_ConnectionId, RequestNumber),
+													   std::vector<IoBuffer>{Request.ReadPayload()});
+		}
 
 		if (!HandlePackageOffers(*Service, Request, m_PackageHandler))
 		{
@@ -891,7 +916,7 @@ HttpAsioServerRequest::TryGetRanges(HttpRanges& Ranges)
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpAsioServerImpl::HttpAsioServerImpl()
+HttpAsioServerImpl::HttpAsioServerImpl() : m_RequestLog(logging::Get("http_requests"))
 {
 }
 
@@ -899,6 +924,12 @@ HttpAsioServerImpl::~HttpAsioServerImpl()
 {
 }
 
+void
+HttpAsioServerImpl::Initialize(std::filesystem::path DataDir)
+{
+	m_RequestTracer.Initialize(DataDir);
+}
+
 int
 HttpAsioServerImpl::Start(uint16_t Port, bool ForceLooopback, int ThreadCount)
 {
@@ -1060,8 +1091,10 @@ HttpAsioServer::RegisterService(HttpService& Service)
 int
 HttpAsioServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
-	ZEN_UNUSED(DataDir);
+	m_Impl->Initialize(DataDir);
+
 	m_BasePort = m_Impl->Start(gsl::narrow<uint16_t>(BasePort), m_ForceLoopback, m_ThreadCount);
+
 	return m_BasePort;
 }
 
diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp
index 3eed9db8f..4a2615133 100644
--- a/src/zenhttp/servers/httpplugin.cpp
+++ b/src/zenhttp/servers/httpplugin.cpp
@@ -2,6 +2,8 @@
 
 #include <zenhttp/httpplugin.h>
 
+#include "httptracer.h"
+
 #if ZEN_WITH_PLUGINS
 
 #	include "httpparser.h"
@@ -103,8 +105,6 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer
 
 	HttpService* RouteRequest(std::string_view Url);
 
-	void WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload);
-
 	struct ServiceEntry
 	{
 		std::string	 ServiceUrlPath;
@@ -119,8 +119,8 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer
 	bool							  m_IsRequestLoggingEnabled = false;
 	LoggerRef						  m_RequestLog;
 	std::atomic_uint32_t			  m_ConnectionIdCounter{0};
-	std::filesystem::path			  m_DataDir;	 // Application data directory
-	std::filesystem::path			  m_PayloadDir;	 // Request debugging payload directory
+
+	HttpServerTracer m_RequestTracer;
 
 	// TransportServer
 
@@ -376,8 +376,8 @@ HttpPluginConnectionHandler::HandleRequest()
 						  ToString(Request.RequestContentType()),
 						  ToString(Request.AcceptContentType()));
 
-			m_Server->WriteDebugPayload(fmt::format("request_{}_{}.bin", m_ConnectionId, RequestNumber),
-										std::vector<IoBuffer>{Request.ReadPayload()});
+			m_Server->m_RequestTracer.WriteDebugPayload(fmt::format("request_{}_{}.bin", m_ConnectionId, RequestNumber),
+														std::vector<IoBuffer>{Request.ReadPayload()});
 		}
 
 		if (!HandlePackageOffers(*Service, Request, m_PackageHandler))
@@ -442,7 +442,8 @@ HttpPluginConnectionHandler::HandleRequest()
 
 			if (m_Server->m_RequestLog.ShouldLog(logging::level::Trace))
 			{
-				m_Server->WriteDebugPayload(fmt::format("response_{}_{}.bin", m_ConnectionId, RequestNumber), ResponseBuffers);
+				m_Server->m_RequestTracer.WriteDebugPayload(fmt::format("response_{}_{}.bin", m_ConnectionId, RequestNumber),
+															ResponseBuffers);
 			}
 
 			for (const IoBuffer& Buffer : ResponseBuffers)
@@ -678,10 +679,7 @@ HttpPluginServerImpl::CreateConnectionHandler(TransportConnection* Connection)
 int
 HttpPluginServerImpl::Initialize(int BasePort, std::filesystem::path DataDir)
 {
-	m_DataDir	 = DataDir;
-	m_PayloadDir = DataDir / "debug" / GetSessionIdString();
-
-	ZEN_INFO("any debug payloads will be written to '{}'", m_PayloadDir);
+	m_RequestTracer.Initialize(DataDir);
 
 	try
 	{
@@ -848,23 +846,6 @@ HttpPluginServerImpl::RouteRequest(std::string_view Url)
 	return CandidateService;
 }
 
-void
-HttpPluginServerImpl::WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload)
-{
-	uint64_t					 PayloadSize = 0;
-	std::vector<const IoBuffer*> Buffers;
-	for (auto& Io : Payload)
-	{
-		Buffers.push_back(&Io);
-		PayloadSize += Io.GetSize();
-	}
-
-	if (PayloadSize)
-	{
-		WriteFile(m_PayloadDir / Filename, Buffers.data(), Buffers.size());
-	}
-}
-
 //////////////////////////////////////////////////////////////////////////
 
 struct HttpPluginServerImpl;
diff --git a/src/zenhttp/servers/httptracer.cpp b/src/zenhttp/servers/httptracer.cpp
new file mode 100644
index 000000000..483307fb1
--- /dev/null
+++ b/src/zenhttp/servers/httptracer.cpp
@@ -0,0 +1,37 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "httptracer.h"
+
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/session.h>
+
+namespace zen {
+
+void
+HttpServerTracer::Initialize(std::filesystem::path DataDir)
+{
+	m_DataDir	 = DataDir;
+	m_PayloadDir = DataDir / "debug" / GetSessionIdString();
+
+	ZEN_INFO("any debug payloads will be written to '{}'", m_PayloadDir);
+}
+
+void
+HttpServerTracer::WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload)
+{
+	uint64_t					 PayloadSize = 0;
+	std::vector<const IoBuffer*> Buffers;
+	for (auto& Io : Payload)
+	{
+		Buffers.push_back(&Io);
+		PayloadSize += Io.GetSize();
+	}
+
+	if (PayloadSize)
+	{
+		WriteFile(m_PayloadDir / Filename, Buffers.data(), Buffers.size());
+	}
+}
+
+}  // namespace zen
diff --git a/src/zenhttp/servers/httptracer.h b/src/zenhttp/servers/httptracer.h
new file mode 100644
index 000000000..da72c79c9
--- /dev/null
+++ b/src/zenhttp/servers/httptracer.h
@@ -0,0 +1,26 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenhttp/httpserver.h>
+
+#pragma once
+
+namespace zen {
+
+/** Helper class for HTTP server implementations
+
+	Provides some common functionality which can be used across all server
+	implementations. These could be in the root class but I think it's nicer
+	to hide the implementation details from client code
+ */
+class HttpServerTracer
+{
+public:
+	void Initialize(std::filesystem::path DataDir);
+	void WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload);
+
+private:
+	std::filesystem::path m_DataDir;	 // Application data directory
+	std::filesystem::path m_PayloadDir;	 // Request debugging payload directory
+};
+
+}  // namespace zen