Merge remote-tracking branch 'origin/main' into sb/rpc-analysis

8 files changed, 590 insertions, 238 deletions

diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index 9fca314b3..2023b6d98 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -1,9 +1,11 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #include "httpasio.h"
+#include "httptracer.h"
 
 #include <zencore/except.h>
 #include <zencore/logging.h>
+#include <zencore/memory/llm.h>
 #include <zencore/thread.h>
 #include <zencore/trace.h>
 #include <zenhttp/httpserver.h>
@@ -31,6 +33,18 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #	define ZEN_TRACE_VERBOSE(fmtstr, ...)
 #endif
 
+namespace zen {
+
+const FLLMTag&
+GetHttpasioTag()
+{
+	static FLLMTag _("httpasio");
+
+	return _;
+}
+
+}  // namespace zen
+
 namespace zen::asio_http {
 
 using namespace std::literals;
@@ -62,6 +76,7 @@ public:
 	HttpAsioServerImpl();
 	~HttpAsioServerImpl();
 
+	void		 Initialize(std::filesystem::path DataDir);
 	int			 Start(uint16_t Port, bool ForceLooopback, int ThreadCount);
 	void		 Stop();
 	void		 RegisterService(const char* UrlPath, HttpService& Service);
@@ -72,6 +87,9 @@ public:
 	std::unique_ptr<asio_http::HttpAcceptor> m_Acceptor;
 	std::vector<std::thread>				 m_ThreadPool;
 
+	LoggerRef		 m_RequestLog;
+	HttpServerTracer m_RequestTracer;
+
 	struct ServiceEntry
 	{
 		std::string	 ServiceUrlPath;
@@ -120,6 +138,8 @@ public:
 
 	void InitializeForPayload(uint16_t ResponseCode, std::span<IoBuffer> BlobList)
 	{
+		ZEN_MEMSCOPE(GetHttpasioTag());
+
 		ZEN_TRACE_CPU("asio::InitializeForPayload");
 
 		m_ResponseCode			  = ResponseCode;
@@ -168,8 +188,8 @@ public:
 		}
 		m_ContentLength = LocalDataSize;
 
-		auto Headers	 = GetHeaders();
-		m_AsioBuffers[0] = asio::const_buffer(Headers.data(), Headers.size());
+		std::string_view Headers = GetHeaders();
+		m_AsioBuffers[0]		 = asio::const_buffer(Headers.data(), Headers.size());
 	}
 
 	uint16_t ResponseCode() const { return m_ResponseCode; }
@@ -179,6 +199,8 @@ public:
 
 	std::string_view GetHeaders()
 	{
+		ZEN_MEMSCOPE(GetHttpasioTag());
+
 		m_Headers << "HTTP/1.1 " << ResponseCode() << " " << ReasonStringForHttpResultCode(ResponseCode()) << "\r\n"
 				  << "Content-Type: " << MapContentTypeToString(m_ContentType) << "\r\n"
 				  << "Content-Length: " << ContentLength() << "\r\n"sv;
@@ -293,7 +315,9 @@ HttpServerConnection::TerminateConnection()
 void
 HttpServerConnection::EnqueueRead()
 {
-	if (m_RequestState == RequestState::kInitialRead)
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
+	if ((m_RequestState == RequestState::kInitialRead) || (m_RequestState == RequestState::kReadingMore))
 	{
 		m_RequestState = RequestState::kReadingMore;
 	}
@@ -313,17 +337,21 @@ HttpServerConnection::EnqueueRead()
 void
 HttpServerConnection::OnDataReceived(const asio::error_code& Ec, [[maybe_unused]] std::size_t ByteCount)
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	if (Ec)
 	{
-		if (m_RequestState == RequestState::kDone || m_RequestState == RequestState::kInitialRead)
+		switch (m_RequestState)
 		{
-			ZEN_TRACE_VERBOSE("on data received ERROR (EXPECTED), connection: {}, reason: '{}'", m_ConnectionId, Ec.message());
-			return;
-		}
-		else
-		{
-			ZEN_WARN("on data received ERROR, connection: {}, reason '{}'", m_ConnectionId, Ec.message());
-			return TerminateConnection();
+			case RequestState::kDone:
+			case RequestState::kInitialRead:
+			case RequestState::kTerminated:
+				ZEN_TRACE_VERBOSE("on data received ERROR (EXPECTED), connection: {}, reason: '{}'", m_ConnectionId, Ec.message());
+				return;
+
+			default:
+				ZEN_WARN("on data received ERROR, connection: {}, reason '{}'", m_ConnectionId, Ec.message());
+				return TerminateConnection();
 		}
 	}
 
@@ -362,6 +390,8 @@ HttpServerConnection::OnDataReceived(const asio::error_code& Ec, [[maybe_unused]
 void
 HttpServerConnection::OnResponseDataSent(const asio::error_code& Ec, [[maybe_unused]] std::size_t ByteCount, bool Pop)
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	if (Ec)
 	{
 		ZEN_WARN("on data sent ERROR, connection: {}, reason: '{}'", m_ConnectionId, Ec.message());
@@ -395,6 +425,8 @@ HttpServerConnection::OnResponseDataSent(const asio::error_code& Ec, [[maybe_unu
 void
 HttpServerConnection::CloseConnection()
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	if (m_RequestState == RequestState::kDone || m_RequestState == RequestState::kTerminated)
 	{
 		return;
@@ -418,6 +450,8 @@ HttpServerConnection::CloseConnection()
 void
 HttpServerConnection::HandleRequest()
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	if (!m_RequestData.IsKeepAlive())
 	{
 		m_RequestState = RequestState::kWritingFinal;
@@ -439,9 +473,29 @@ HttpServerConnection::HandleRequest()
 	{
 		ZEN_TRACE_CPU("asio::HandleRequest");
 
+		const uint32_t RequestNumber = m_RequestCounter.load(std::memory_order_relaxed);
+
 		HttpAsioServerRequest Request(m_RequestData, *Service, m_RequestData.Body());
 
-		ZEN_TRACE_VERBOSE("handle request, connection: {}, request: {}'", m_ConnectionId, m_RequestCounter.load(std::memory_order_relaxed));
+		ZEN_TRACE_VERBOSE("handle request, connection: {}, request: {}'", m_ConnectionId, RequestNumber);
+
+		const HttpVerb		   RequestVerb = Request.RequestVerb();
+		const std::string_view Uri		   = Request.RelativeUri();
+
+		if (m_Server.m_RequestLog.ShouldLog(logging::level::Trace))
+		{
+			ZEN_LOG_TRACE(m_Server.m_RequestLog,
+						  "connection #{} Handling Request: {} {} ({} bytes ({}), accept: {})",
+						  m_ConnectionId,
+						  ToString(RequestVerb),
+						  Uri,
+						  Request.ContentLength(),
+						  ToString(Request.RequestContentType()),
+						  ToString(Request.AcceptContentType()));
+
+			m_Server.m_RequestTracer.WriteDebugPayload(fmt::format("request_{}_{}.bin", m_ConnectionId, RequestNumber),
+													   std::vector<IoBuffer>{Request.ReadPayload()});
+		}
 
 		if (!HandlePackageOffers(*Service, Request, m_PackageHandler))
 		{
@@ -449,7 +503,15 @@ HttpServerConnection::HandleRequest()
 			{
 				Service->HandleRequest(Request);
 			}
-			catch (std::system_error& SystemError)
+			catch (const AssertException& AssertEx)
+			{
+				// Drop any partially formatted response
+				Request.m_Response.reset();
+
+				ZEN_ERROR("Caught assert exception while handling request: {}", AssertEx.FullDescription());
+				Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, AssertEx.FullDescription());
+			}
+			catch (const std::system_error& SystemError)
 			{
 				// Drop any partially formatted response
 				Request.m_Response.reset();
@@ -460,23 +522,25 @@ HttpServerConnection::HandleRequest()
 				}
 				else
 				{
-					ZEN_ERROR("Caught system error exception while handling request: {}", SystemError.what());
+					ZEN_WARN("Caught system error exception while handling request: {}. ({})",
+							 SystemError.what(),
+							 SystemError.code().value());
 					Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, SystemError.what());
 				}
 			}
-			catch (std::bad_alloc& BadAlloc)
+			catch (const std::bad_alloc& BadAlloc)
 			{
 				// Drop any partially formatted response
 				Request.m_Response.reset();
 
 				Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, BadAlloc.what());
 			}
-			catch (std::exception& ex)
+			catch (const std::exception& ex)
 			{
 				// Drop any partially formatted response
 				Request.m_Response.reset();
 
-				ZEN_ERROR("Caught exception while handling request: {}", ex.what());
+				ZEN_WARN("Caught exception while handling request: {}", ex.what());
 				Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, ex.what());
 			}
 		}
@@ -490,11 +554,11 @@ HttpServerConnection::HandleRequest()
 				Response->SuppressPayload();
 			}
 
-			auto ResponseBuffers = Response->AsioBuffers();
+			const std::vector<asio::const_buffer>& ResponseBuffers = Response->AsioBuffers();
 
 			uint64_t ResponseLength = 0;
 
-			for (auto& Buffer : ResponseBuffers)
+			for (const asio::const_buffer& Buffer : ResponseBuffers)
 			{
 				ResponseLength += Buffer.size();
 			}
@@ -573,29 +637,49 @@ struct HttpAcceptor
 	: m_Server(Server)
 	, m_IoService(IoService)
 	, m_Acceptor(m_IoService, asio::ip::tcp::v6())
+	, m_AlternateProtocolAcceptor(m_IoService, asio::ip::tcp::v4())
 	{
 		m_Acceptor.set_option(asio::ip::v6_only(false));
 #if ZEN_PLATFORM_WINDOWS
 		// Special option for Windows settings as !asio::socket_base::reuse_address is not the same as exclusive access on Windows platforms
-		typedef asio::detail::socket_option::boolean<ASIO_OS_DEF(SOL_SOCKET), SO_EXCLUSIVEADDRUSE> excluse_address;
-		m_Acceptor.set_option(excluse_address(true));
+		typedef asio::detail::socket_option::boolean<ASIO_OS_DEF(SOL_SOCKET), SO_EXCLUSIVEADDRUSE> exclusive_address;
+		m_Acceptor.set_option(exclusive_address(true));
+		m_AlternateProtocolAcceptor.set_option(exclusive_address(true));
 #else	// ZEN_PLATFORM_WINDOWS
 		m_Acceptor.set_option(asio::socket_base::reuse_address(false));
+		m_AlternateProtocolAcceptor.set_option(asio::socket_base::reuse_address(false));
 #endif	// ZEN_PLATFORM_WINDOWS
 
 		m_Acceptor.set_option(asio::ip::tcp::no_delay(true));
 		m_Acceptor.set_option(asio::socket_base::receive_buffer_size(128 * 1024));
 		m_Acceptor.set_option(asio::socket_base::send_buffer_size(256 * 1024));
 
+		m_AlternateProtocolAcceptor.set_option(asio::ip::tcp::no_delay(true));
+		m_AlternateProtocolAcceptor.set_option(asio::socket_base::receive_buffer_size(128 * 1024));
+		m_AlternateProtocolAcceptor.set_option(asio::socket_base::send_buffer_size(256 * 1024));
+
 		asio::ip::address_v6 BindAddress   = ForceLoopback ? asio::ip::address_v6::loopback() : asio::ip::address_v6::any();
 		uint16_t			 EffectivePort = BasePort;
 
+		if (BindAddress.is_loopback())
+		{
+			m_Acceptor.set_option(asio::ip::v6_only(true));
+		}
+
 		asio::error_code BindErrorCode;
 		m_Acceptor.bind(asio::ip::tcp::endpoint(BindAddress, EffectivePort), BindErrorCode);
 		if (BindErrorCode == asio::error::access_denied && !BindAddress.is_loopback())
 		{
 			// Access denied for a public port - lets try fall back to local port only
 			BindAddress = asio::ip::address_v6::loopback();
+			m_Acceptor.set_option(asio::ip::v6_only(true));
+			m_Acceptor.bind(asio::ip::tcp::endpoint(BindAddress, EffectivePort), BindErrorCode);
+		}
+		if (BindErrorCode == asio::error::address_in_use)
+		{
+			// Do a retry after a short sleep on same port just to be sure
+			ZEN_INFO("Desired port {} is in use, retrying", BasePort);
+			Sleep(100);
 			m_Acceptor.bind(asio::ip::tcp::endpoint(BindAddress, EffectivePort), BindErrorCode);
 		}
 		// Sharing violation implies the port is being used by another process
@@ -613,9 +697,20 @@ struct HttpAcceptor
 		{
 			ZEN_ERROR("Unable open asio service, error '{}'", BindErrorCode.message());
 		}
-		else if (BindAddress.is_loopback())
+		else
 		{
-			ZEN_INFO("Registered local-only handler 'http://{}:{}/' - this is not accessible from remote hosts", "[::1]", EffectivePort);
+			if (EffectivePort != BasePort)
+			{
+				ZEN_WARN("Desired port {} is in use, remapped to port {}", BasePort, EffectivePort);
+			}
+			if (BindAddress.is_loopback())
+			{
+				m_AlternateProtocolAcceptor.bind(asio::ip::tcp::endpoint(asio::ip::address_v4::loopback(), EffectivePort), BindErrorCode);
+				m_UseAlternateProtocolAcceptor = true;
+				ZEN_INFO("Registered local-only handler 'http://{}:{}/' - this is not accessible from remote hosts",
+						 "localhost",
+						 EffectivePort);
+			}
 		}
 
 #if ZEN_PLATFORM_WINDOWS
@@ -635,31 +730,66 @@ struct HttpAcceptor
 				 &OptionNumberOfBytesReturned,
 				 0,
 				 0);
+		if (m_UseAlternateProtocolAcceptor)
+		{
+			NativeSocket = m_AlternateProtocolAcceptor.native_handle();
+			WSAIoctl(NativeSocket,
+					 SIO_LOOPBACK_FAST_PATH,
+					 &LoopbackOptionValue,
+					 sizeof(LoopbackOptionValue),
+					 NULL,
+					 0,
+					 &OptionNumberOfBytesReturned,
+					 0,
+					 0);
+		}
 #endif
 		m_Acceptor.listen();
+		if (m_UseAlternateProtocolAcceptor)
+		{
+			m_AlternateProtocolAcceptor.listen();
+		}
 
 		ZEN_INFO("Started asio server at 'http://{}:{}'", BindAddress.is_loopback() ? "[::1]" : "*", EffectivePort);
 	}
 
+	~HttpAcceptor()
+	{
+		m_Acceptor.close();
+		if (m_UseAlternateProtocolAcceptor)
+		{
+			m_AlternateProtocolAcceptor.close();
+		}
+	}
+
 	void Start()
 	{
-		m_Acceptor.listen();
-		InitAccept();
+		ZEN_MEMSCOPE(GetHttpasioTag());
+
+		ZEN_ASSERT(!m_IsStopped);
+		InitAcceptInternal(m_Acceptor);
+		if (m_UseAlternateProtocolAcceptor)
+		{
+			InitAcceptInternal(m_AlternateProtocolAcceptor);
+		}
 	}
 
-	void Stop() { m_IsStopped = true; }
+	void StopAccepting() { m_IsStopped = true; }
+
+	int GetAcceptPort() { return m_Acceptor.local_endpoint().port(); }
 
-	void InitAccept()
+private:
+	void InitAcceptInternal(asio::ip::tcp::acceptor& Acceptor)
 	{
 		auto				   SocketPtr = std::make_unique<asio::ip::tcp::socket>(m_IoService);
 		asio::ip::tcp::socket& SocketRef = *SocketPtr.get();
 
-		m_Acceptor.async_accept(SocketRef, [this, Socket = std::move(SocketPtr)](const asio::error_code& Ec) mutable {
+		Acceptor.async_accept(SocketRef, [this, &Acceptor, Socket = std::move(SocketPtr)](const asio::error_code& Ec) mutable {
 			if (Ec)
 			{
 				ZEN_WARN("asio async_accept, connection failed to '{}:{}' reason '{}'",
-						 m_Acceptor.local_endpoint().address().to_string(),
-						 m_Acceptor.local_endpoint().port(),
+						 Acceptor.local_endpoint().address().to_string(),
+						 Acceptor.local_endpoint().port(),
 						 Ec.message());
 			}
 			else
@@ -679,12 +809,12 @@ struct HttpAcceptor
 
 			if (!m_IsStopped.load())
 			{
-				InitAccept();
+				InitAcceptInternal(Acceptor);
 			}
 			else
 			{
 				std::error_code CloseEc;
-				m_Acceptor.close(CloseEc);
+				Acceptor.close(CloseEc);
 				if (CloseEc)
 				{
 					ZEN_WARN("acceptor close ERROR, reason '{}'", CloseEc.message());
@@ -693,12 +823,11 @@ struct HttpAcceptor
 		});
 	}
 
-	int GetAcceptPort() { return m_Acceptor.local_endpoint().port(); }
-
-private:
 	HttpAsioServerImpl&		m_Server;
 	asio::io_service&		m_IoService;
 	asio::ip::tcp::acceptor m_Acceptor;
+	asio::ip::tcp::acceptor m_AlternateProtocolAcceptor;
+	bool					m_UseAlternateProtocolAcceptor{false};
 	std::atomic<bool>		m_IsStopped{false};
 };
 
@@ -787,6 +916,8 @@ HttpAsioServerRequest::ReadPayload()
 void
 HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode)
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	ZEN_ASSERT(!m_Response);
 
 	m_Response.reset(new HttpResponse(HttpContentType::kBinary));
@@ -798,6 +929,8 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode)
 void
 HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs)
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	ZEN_ASSERT(!m_Response);
 
 	m_Response.reset(new HttpResponse(ContentType));
@@ -807,6 +940,8 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT
 void
 HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::u8string_view ResponseString)
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	ZEN_ASSERT(!m_Response);
 	m_Response.reset(new HttpResponse(ContentType));
 
@@ -819,6 +954,8 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT
 void
 HttpAsioServerRequest::WriteResponseAsync(std::function<void(HttpServerRequest&)>&& ContinuationHandler)
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	ZEN_ASSERT(!m_Response);
 
 	// Not one bit async, innit
@@ -833,7 +970,7 @@ HttpAsioServerRequest::TryGetRanges(HttpRanges& Ranges)
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpAsioServerImpl::HttpAsioServerImpl()
+HttpAsioServerImpl::HttpAsioServerImpl() : m_RequestLog(logging::Get("http_requests"))
 {
 }
 
@@ -841,9 +978,17 @@ HttpAsioServerImpl::~HttpAsioServerImpl()
 {
 }
 
+void
+HttpAsioServerImpl::Initialize(std::filesystem::path DataDir)
+{
+	m_RequestTracer.Initialize(DataDir);
+}
+
 int
 HttpAsioServerImpl::Start(uint16_t Port, bool ForceLooopback, int ThreadCount)
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	ZEN_ASSERT(ThreadCount > 0);
 
 	ZEN_INFO("starting asio http with {} service threads", ThreadCount);
@@ -863,13 +1008,19 @@ HttpAsioServerImpl::Start(uint16_t Port, bool ForceLooopback, int ThreadCount)
 	for (int i = 0; i < ThreadCount; ++i)
 	{
 		m_ThreadPool.emplace_back([this, Index = i + 1] {
+			ZEN_MEMSCOPE(GetHttpasioTag());
+
 			SetCurrentThreadName(fmt::format("asio_io_{}", Index));
 
 			try
 			{
 				m_IoService.run();
 			}
-			catch (std::exception& e)
+			catch (const AssertException& AssertEx)
+			{
+				ZEN_ERROR("Assert caught in asio event loop: {}", AssertEx.FullDescription());
+			}
+			catch (const std::exception& e)
 			{
 				ZEN_ERROR("Exception caught in asio event loop: '{}'", e.what());
 			}
@@ -884,17 +1035,29 @@ HttpAsioServerImpl::Start(uint16_t Port, bool ForceLooopback, int ThreadCount)
 void
 HttpAsioServerImpl::Stop()
 {
-	m_Acceptor->Stop();
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
+	if (m_Acceptor)
+	{
+		m_Acceptor->StopAccepting();
+	}
 	m_IoService.stop();
 	for (auto& Thread : m_ThreadPool)
 	{
-		Thread.join();
+		if (Thread.joinable())
+		{
+			Thread.join();
+		}
 	}
+	m_ThreadPool.clear();
+	m_Acceptor.reset();
 }
 
 void
 HttpAsioServerImpl::RegisterService(const char* InUrlPath, HttpService& Service)
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	std::string_view UrlPath(InUrlPath);
 	Service.SetUriPrefixLength(UrlPath.size());
 	if (!UrlPath.empty() && UrlPath.back() == '/')
@@ -909,6 +1072,8 @@ HttpAsioServerImpl::RegisterService(const char* InUrlPath, HttpService& Service)
 HttpService*
 HttpAsioServerImpl::RouteRequest(std::string_view Url)
 {
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	RwLock::SharedLockScope _(m_Lock);
 
 	HttpService*		   CandidateService	  = nullptr;
@@ -978,7 +1143,7 @@ HttpAsioServer::Close()
 	{
 		m_Impl->Stop();
 	}
-	catch (std::exception& ex)
+	catch (const std::exception& ex)
 	{
 		ZEN_WARN("Caught exception stopping http asio server: {}", ex.what());
 	}
@@ -994,8 +1159,11 @@ HttpAsioServer::RegisterService(HttpService& Service)
 int
 HttpAsioServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
-	ZEN_UNUSED(DataDir);
+	ZEN_TRACE_CPU("HttpAsioServer::Initialize");
+	m_Impl->Initialize(DataDir);
+
 	m_BasePort = m_Impl->Start(gsl::narrow<uint16_t>(BasePort), m_ForceLoopback, m_ThreadCount);
+
 	return m_BasePort;
 }
 
@@ -1052,6 +1220,9 @@ HttpAsioServer::RequestExit()
 Ref<HttpServer>
 CreateHttpAsioServer(bool ForceLoopback, unsigned int ThreadCount)
 {
+	ZEN_TRACE_CPU("CreateHttpAsioServer");
+	ZEN_MEMSCOPE(GetHttpasioTag());
+
 	return Ref<HttpServer>{new HttpAsioServer(ForceLoopback, ThreadCount)};
 }
 
diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp
index 2a6a90d2e..b8b7931a9 100644
--- a/src/zenhttp/servers/httpmulti.cpp
+++ b/src/zenhttp/servers/httpmulti.cpp
@@ -3,6 +3,7 @@
 #include "httpmulti.h"
 
 #include <zencore/logging.h>
+#include <zencore/trace.h>
 
 #if ZEN_PLATFORM_WINDOWS
 #	include <conio.h>
@@ -30,6 +31,8 @@ HttpMultiServer::RegisterService(HttpService& Service)
 int
 HttpMultiServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_TRACE_CPU("HttpMultiServer::Initialize");
+
 	ZEN_UNUSED(DataDir);
 	ZEN_ASSERT(!m_IsInitialized);
 
@@ -103,6 +106,10 @@ HttpMultiServer::RequestExit()
 void
 HttpMultiServer::Close()
 {
+	for (auto& Server : m_Servers)
+	{
+		Server->Close();
+	}
 }
 
 void
diff --git a/src/zenhttp/servers/httpparser.cpp b/src/zenhttp/servers/httpparser.cpp
index c64134c95..93094e21b 100644
--- a/src/zenhttp/servers/httpparser.cpp
+++ b/src/zenhttp/servers/httpparser.cpp
@@ -6,6 +6,8 @@
 #include <zencore/logging.h>
 #include <zencore/string.h>
 
+#include <limits>
+
 namespace zen {
 
 using namespace std::literals;
@@ -69,23 +71,21 @@ HttpRequestParser::ConsumeData(const char* InputData, size_t DataSize)
 int
 HttpRequestParser::OnUrl(const char* Data, size_t Bytes)
 {
-	if (!m_Url)
+	const size_t RemainingBufferSpace = std::numeric_limits<std::uint32_t>::max() - m_HeaderData.size();
+	if (RemainingBufferSpace < Bytes)
 	{
-		ZEN_ASSERT_SLOW(m_UrlLength == 0);
-		m_Url = m_HeaderCursor;
+		ZEN_WARN("HTTP parser does not have enough space for incoming request headers, need {} more bytes", Bytes - RemainingBufferSpace);
+		return 1;
 	}
 
-	const size_t RemainingBufferSpace = sizeof m_HeaderBuffer + m_HeaderBuffer - m_HeaderCursor;
-
-	if (RemainingBufferSpace < Bytes)
+	if (m_UrlRange.Length == 0)
 	{
-		ZEN_WARN("HTTP parser does not have enough space for incoming request, need {} more bytes", Bytes - RemainingBufferSpace);
-		return 1;
+		ZEN_ASSERT_SLOW(m_UrlRange.Offset == 0);
+		m_UrlRange.Offset = (uint32_t)m_HeaderData.size();
 	}
 
-	memcpy(m_HeaderCursor, Data, Bytes);
-	m_HeaderCursor += Bytes;
-	m_UrlLength += Bytes;
+	m_HeaderData.insert(m_HeaderData.end(), Data, &Data[Bytes]);
+	m_UrlRange.Length += (uint32_t)Bytes;
 
 	return 0;
 }
@@ -93,56 +93,70 @@ HttpRequestParser::OnUrl(const char* Data, size_t Bytes)
 int
 HttpRequestParser::OnHeader(const char* Data, size_t Bytes)
 {
-	if (m_CurrentHeaderValueLength)
+	const size_t RemainingBufferSpace = std::numeric_limits<std::uint32_t>::max() - m_HeaderData.size();
+	if (RemainingBufferSpace < Bytes)
 	{
-		AppendCurrentHeader();
+		ZEN_WARN("HTTP parser does not have enough space for incoming request headers, need {} more bytes", Bytes - RemainingBufferSpace);
+		return 1;
+	}
 
-		m_CurrentHeaderNameLength  = 0;
-		m_CurrentHeaderValueLength = 0;
-		m_CurrentHeaderName		   = m_HeaderCursor;
+	if (m_HeaderEntries.empty())
+	{
+		m_HeaderEntries.resize(1);
 	}
-	else if (m_CurrentHeaderName == nullptr)
+	HeaderEntry* CurrentHeaderEntry = &m_HeaderEntries.back();
+	if (CurrentHeaderEntry->ValueRange.Length)
 	{
-		m_CurrentHeaderName = m_HeaderCursor;
+		ParseCurrentHeader();
+		m_HeaderEntries.emplace_back(HeaderEntry{.NameRange = {.Offset = (uint32_t)m_HeaderData.size()}});
+		CurrentHeaderEntry = &m_HeaderEntries.back();
 	}
-
-	const size_t RemainingBufferSpace = sizeof m_HeaderBuffer + m_HeaderBuffer - m_HeaderCursor;
-	if (RemainingBufferSpace < Bytes)
+	else if (CurrentHeaderEntry->NameRange.Length == 0)
 	{
-		ZEN_WARN("HTTP parser does not have enough space for incoming header name, need {} more bytes", Bytes - RemainingBufferSpace);
-		return 1;
+		m_HeaderEntries.emplace_back(HeaderEntry{.NameRange = {.Offset = (uint32_t)m_HeaderData.size()}});
+		CurrentHeaderEntry = &m_HeaderEntries.back();
 	}
 
-	memcpy(m_HeaderCursor, Data, Bytes);
-	m_HeaderCursor += Bytes;
-	m_CurrentHeaderNameLength += Bytes;
+	m_HeaderData.insert(m_HeaderData.end(), Data, &Data[Bytes]);
+	CurrentHeaderEntry->NameRange.Length += (uint32_t)Bytes;
 
 	return 0;
 }
 
 void
-HttpRequestParser::AppendCurrentHeader()
+HttpRequestParser::ParseCurrentHeader()
 {
-	std::string_view HeaderName(m_CurrentHeaderName, m_CurrentHeaderNameLength);
-	std::string_view HeaderValue(m_CurrentHeaderValue, m_CurrentHeaderValueLength);
+	ZEN_ASSERT_SLOW(!m_HeaderEntries.empty());
+	const HeaderEntry&	   CurrentHeaderEntry = m_HeaderEntries.back();
+	const size_t		   CurrentHeaderCount = m_HeaderEntries.size();
+	const std::string_view HeaderName(GetHeaderSubString(CurrentHeaderEntry.NameRange));
+	if (CurrentHeaderCount > std::numeric_limits<int8_t>::max())
+	{
+		ZEN_WARN("HttpRequestParser parser only supports up to {} headers, can't store header '{}'. Dropping it.",
+				 std::numeric_limits<int8_t>::max(),
+				 HeaderName);
+		return;
+	}
+	const std::string_view HeaderValue(GetHeaderSubString(CurrentHeaderEntry.ValueRange));
 
-	const uint32_t HeaderHash = HashStringAsLowerDjb2(HeaderName);
+	const uint32_t HeaderHash		  = HashStringAsLowerDjb2(HeaderName);
+	const int8_t   CurrentHeaderIndex = int8_t(CurrentHeaderCount - 1);
 
 	if (HeaderHash == HashContentLength)
 	{
-		m_ContentLengthHeaderIndex = (int8_t)m_Headers.size();
+		m_ContentLengthHeaderIndex = CurrentHeaderIndex;
 	}
 	else if (HeaderHash == HashAccept)
 	{
-		m_AcceptHeaderIndex = (int8_t)m_Headers.size();
+		m_AcceptHeaderIndex = CurrentHeaderIndex;
 	}
 	else if (HeaderHash == HashContentType)
 	{
-		m_ContentTypeHeaderIndex = (int8_t)m_Headers.size();
+		m_ContentTypeHeaderIndex = CurrentHeaderIndex;
 	}
 	else if (HeaderHash == HashSession)
 	{
-		m_SessionId = Oid::FromHexString(HeaderValue);
+		m_SessionId = Oid::TryFromHexString(HeaderValue);
 	}
 	else if (HeaderHash == HashRequest)
 	{
@@ -162,38 +176,38 @@ HttpRequestParser::AppendCurrentHeader()
 	}
 	else if (HeaderHash == HashRange)
 	{
-		m_RangeHeaderIndex = (int8_t)m_Headers.size();
+		m_RangeHeaderIndex = CurrentHeaderIndex;
 	}
-
-	m_Headers.emplace_back(HeaderName, HeaderValue);
 }
 
 int
 HttpRequestParser::OnHeaderValue(const char* Data, size_t Bytes)
 {
-	if (m_CurrentHeaderValueLength == 0)
-	{
-		m_CurrentHeaderValue = m_HeaderCursor;
-	}
-
-	const size_t RemainingBufferSpace = sizeof m_HeaderBuffer + m_HeaderBuffer - m_HeaderCursor;
+	const size_t RemainingBufferSpace = std::numeric_limits<std::uint32_t>::max() - m_HeaderData.size();
 	if (RemainingBufferSpace < Bytes)
 	{
-		ZEN_WARN("HTTP parser does not have enough space for incoming header value, need {} more bytes", Bytes - RemainingBufferSpace);
+		ZEN_WARN("HTTP parser does not have enough space for incoming request headers, need {} more bytes", Bytes - RemainingBufferSpace);
 		return 1;
 	}
 
-	memcpy(m_HeaderCursor, Data, Bytes);
-	m_HeaderCursor += Bytes;
-	m_CurrentHeaderValueLength += Bytes;
+	ZEN_ASSERT_SLOW(!m_HeaderEntries.empty());
+	HeaderEntry& CurrentHeaderEntry = m_HeaderEntries.back();
+	if (CurrentHeaderEntry.ValueRange.Length == 0)
+	{
+		CurrentHeaderEntry.ValueRange.Offset = (uint32_t)m_HeaderData.size();
+	}
 
+	m_HeaderData.insert(m_HeaderData.end(), Data, &Data[Bytes]);
+	CurrentHeaderEntry.ValueRange.Length += (uint32_t)Bytes;
 	return 0;
 }
 
 static void
-NormalizeUrlPath(const char* Url, size_t UrlLength, std::string& NormalizedUrl)
+NormalizeUrlPath(std::string_view InUrl, std::string& NormalizedUrl)
 {
-	bool LastCharWasSeparator = false;
+	bool		 LastCharWasSeparator = false;
+	const char*	 Url				  = InUrl.data();
+	const size_t UrlLength			  = InUrl.length();
 	for (std::string_view::size_type UrlIndex = 0; UrlIndex < UrlLength; ++UrlIndex)
 	{
 		const char UrlChar	   = Url[UrlIndex];
@@ -226,9 +240,13 @@ HttpRequestParser::OnHeadersComplete()
 {
 	try
 	{
-		if (m_CurrentHeaderValueLength)
+		if (!m_HeaderEntries.empty())
 		{
-			AppendCurrentHeader();
+			HeaderEntry& CurrentHeaderEntry = m_HeaderEntries.back();
+			if (CurrentHeaderEntry.NameRange.Length)
+			{
+				ParseCurrentHeader();
+			}
 		}
 
 		m_KeepAlive = !!http_should_keep_alive(&m_Parser);
@@ -268,21 +286,21 @@ HttpRequestParser::OnHeadersComplete()
 				break;
 		}
 
-		std::string_view Url(m_Url, m_UrlLength);
+		std::string_view FullUrl(GetHeaderSubString(m_UrlRange));
 
-		if (auto QuerySplit = Url.find_first_of('?'); QuerySplit != std::string_view::npos)
+		if (auto QuerySplit = FullUrl.find_first_of('?'); QuerySplit != std::string_view::npos)
 		{
-			m_UrlLength	  = QuerySplit;
-			m_QueryString = m_Url + QuerySplit + 1;
-			m_QueryLength = Url.size() - QuerySplit - 1;
+			m_UrlRange.Length  = uint32_t(QuerySplit);
+			m_QueryStringRange = {.Offset = uint32_t(m_UrlRange.Offset + QuerySplit + 1),
+								  .Length = uint32_t(FullUrl.size() - QuerySplit - 1)};
 		}
 
-		NormalizeUrlPath(m_Url, m_UrlLength, m_NormalizedUrl);
+		NormalizeUrlPath(FullUrl, m_NormalizedUrl);
 
-		if (m_ContentLengthHeaderIndex >= 0)
+		std::string_view Value = GetHeaderValue(m_ContentLengthHeaderIndex);
+		if (!Value.empty())
 		{
-			std::string_view& Value			= m_Headers[m_ContentLengthHeaderIndex].Value;
-			uint64_t		  ContentLength = 0;
+			uint64_t ContentLength = 0;
 			std::from_chars(Value.data(), Value.data() + Value.size(), ContentLength);
 
 			if (ContentLength)
@@ -330,16 +348,11 @@ HttpRequestParser::OnBody(const char* Data, size_t Bytes)
 void
 HttpRequestParser::ResetState()
 {
-	m_HeaderCursor			   = m_HeaderBuffer;
-	m_CurrentHeaderName		   = nullptr;
-	m_CurrentHeaderNameLength  = 0;
-	m_CurrentHeaderValue	   = nullptr;
-	m_CurrentHeaderValueLength = 0;
-	m_CurrentHeaderName		   = nullptr;
-	m_Url					   = nullptr;
-	m_UrlLength				   = 0;
-	m_QueryString			   = nullptr;
-	m_QueryLength			   = 0;
+	m_UrlRange		   = {};
+	m_QueryStringRange = {};
+
+	m_HeaderEntries.clear();
+
 	m_ContentLengthHeaderIndex = -1;
 	m_AcceptHeaderIndex		   = -1;
 	m_ContentTypeHeaderIndex   = -1;
@@ -347,7 +360,8 @@ HttpRequestParser::ResetState()
 	m_Expect100Continue		   = false;
 	m_BodyBuffer			   = {};
 	m_BodyPosition			   = 0;
-	m_Headers.clear();
+
+	m_HeaderData.clear();
 	m_NormalizedUrl.clear();
 }
 
@@ -366,7 +380,12 @@ HttpRequestParser::OnMessageComplete()
 		ResetState();
 		return 0;
 	}
-	catch (std::system_error& SystemError)
+	catch (const AssertException& AssertEx)
+	{
+		ZEN_WARN("Assert caught when processing http request: {}", AssertEx.FullDescription());
+		return 1;
+	}
+	catch (const std::system_error& SystemError)
 	{
 		if (IsOOM(SystemError.code()))
 		{
@@ -378,18 +397,18 @@ HttpRequestParser::OnMessageComplete()
 		}
 		else
 		{
-			ZEN_ERROR("failed processing http request: '{}'", SystemError.what());
+			ZEN_ERROR("failed processing http request: '{}' ({})", SystemError.what(), SystemError.code().value());
 		}
 		ResetState();
 		return 1;
 	}
-	catch (std::bad_alloc& BadAlloc)
+	catch (const std::bad_alloc& BadAlloc)
 	{
 		ZEN_WARN("out of memory when processing http request: '{}'", BadAlloc.what());
 		ResetState();
 		return 1;
 	}
-	catch (std::exception& Ex)
+	catch (const std::exception& Ex)
 	{
 		ZEN_ERROR("failed processing http request: '{}'", Ex.what());
 		ResetState();
diff --git a/src/zenhttp/servers/httpparser.h b/src/zenhttp/servers/httpparser.h
index bdbcab4d9..0d2664ec5 100644
--- a/src/zenhttp/servers/httpparser.h
+++ b/src/zenhttp/servers/httpparser.h
@@ -5,6 +5,8 @@
 #include <zencore/uid.h>
 #include <zenhttp/httpcommon.h>
 
+#include <EASTL/fixed_vector.h>
+
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <http_parser.h>
 ZEN_THIRD_PARTY_INCLUDES_END
@@ -31,73 +33,68 @@ struct HttpRequestParser
 
 	HttpVerb		 RequestVerb() const { return m_RequestVerb; }
 	bool			 IsKeepAlive() const { return m_KeepAlive; }
-	std::string_view Url() const { return m_NormalizedUrl.empty() ? std::string_view(m_Url, m_UrlLength) : m_NormalizedUrl; }
-	std::string_view QueryString() const { return std::string_view(m_QueryString, m_QueryLength); }
+	std::string_view Url() const { return m_NormalizedUrl.empty() ? GetHeaderSubString(m_UrlRange) : m_NormalizedUrl; }
+	std::string_view QueryString() const { return GetHeaderSubString(m_QueryStringRange); }
 	IoBuffer		 Body() { return m_BodyBuffer; }
 
-	inline HttpContentType ContentType()
-	{
-		if (m_ContentTypeHeaderIndex < 0)
-		{
-			return HttpContentType::kUnknownContentType;
-		}
-
-		return ParseContentType(m_Headers[m_ContentTypeHeaderIndex].Value);
-	}
+	inline HttpContentType ContentType() { return ParseContentType(GetHeaderValue(m_ContentTypeHeaderIndex)); }
 
-	inline HttpContentType AcceptType()
-	{
-		if (m_AcceptHeaderIndex < 0)
-		{
-			return HttpContentType::kUnknownContentType;
-		}
-
-		return ParseContentType(m_Headers[m_AcceptHeaderIndex].Value);
-	}
+	inline HttpContentType AcceptType() { return ParseContentType(GetHeaderValue(m_AcceptHeaderIndex)); }
 
 	Oid SessionId() const { return m_SessionId; }
 	int RequestId() const { return m_RequestId; }
 
-	std::string_view RangeHeader() const { return m_RangeHeaderIndex != -1 ? m_Headers[m_RangeHeaderIndex].Value : std::string_view(); }
+	std::string_view RangeHeader() const { return GetHeaderValue(m_RangeHeaderIndex); }
 
 private:
+	struct HeaderRange
+	{
+		uint32_t Offset = 0;
+		uint32_t Length = 0;
+	};
+
 	struct HeaderEntry
 	{
-		HeaderEntry() = default;
+		HeaderRange NameRange;
+		HeaderRange ValueRange;
+	};
 
-		HeaderEntry(std::string_view InName, std::string_view InValue) : Name(InName), Value(InValue) {}
+	inline std::string_view GetHeaderValue(int8_t HeaderIndex) const
+	{
+		if (HeaderIndex == -1)
+		{
+			return {};
+		}
+		ZEN_ASSERT(size_t(HeaderIndex) < m_HeaderEntries.size());
+		return GetHeaderSubString(m_HeaderEntries[HeaderIndex].ValueRange);
+	}
 
-		std::string_view Name;
-		std::string_view Value;
-	};
+	std::string_view GetHeaderSubString(const HeaderRange& Range) const
+	{
+		ZEN_ASSERT_SLOW(Range.Offset + Range.Length <= m_HeaderData.size());
+		return std::string_view(m_HeaderData.begin(), m_HeaderData.size()).substr(Range.Offset, Range.Length);
+	}
 
-	HttpRequestParserCallbacks& m_Connection;
-	char*						m_HeaderCursor			   = m_HeaderBuffer;
-	char*						m_Url					   = nullptr;
-	size_t						m_UrlLength				   = 0;
-	char*						m_QueryString			   = nullptr;
-	size_t						m_QueryLength			   = 0;
-	char*						m_CurrentHeaderName		   = nullptr;  // Used while parsing headers
-	size_t						m_CurrentHeaderNameLength  = 0;
-	char*						m_CurrentHeaderValue	   = nullptr;  // Used while parsing headers
-	size_t						m_CurrentHeaderValueLength = 0;
-	std::vector<HeaderEntry>	m_Headers;
-	int8_t						m_ContentLengthHeaderIndex;
-	int8_t						m_AcceptHeaderIndex;
-	int8_t						m_ContentTypeHeaderIndex;
-	int8_t						m_RangeHeaderIndex;
-	HttpVerb					m_RequestVerb;
-	std::atomic_bool			m_KeepAlive{false};
-	bool						m_Expect100Continue = false;
-	int							m_RequestId			= -1;
-	Oid							m_SessionId{};
-	IoBuffer					m_BodyBuffer;
-	uint64_t					m_BodyPosition = 0;
-	http_parser					m_Parser;
-	char						m_HeaderBuffer[1024];
-	std::string					m_NormalizedUrl;
-
-	void AppendCurrentHeader();
+	HttpRequestParserCallbacks&			 m_Connection;
+	HeaderRange							 m_UrlRange;
+	HeaderRange							 m_QueryStringRange;
+	eastl::fixed_vector<HeaderEntry, 16> m_HeaderEntries;
+	int8_t								 m_ContentLengthHeaderIndex;
+	int8_t								 m_AcceptHeaderIndex;
+	int8_t								 m_ContentTypeHeaderIndex;
+	int8_t								 m_RangeHeaderIndex;
+	HttpVerb							 m_RequestVerb;
+	std::atomic_bool					 m_KeepAlive{false};
+	bool								 m_Expect100Continue = false;
+	int									 m_RequestId		 = -1;
+	Oid									 m_SessionId{};
+	IoBuffer							 m_BodyBuffer;
+	uint64_t							 m_BodyPosition = 0;
+	http_parser							 m_Parser;
+	eastl::fixed_vector<char, 512>		 m_HeaderData;
+	std::string							 m_NormalizedUrl;
+
+	void ParseCurrentHeader();
 
 	int OnMessageBegin();
 	int OnUrl(const char* Data, size_t Bytes);
diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp
index 3eed9db8f..d6ca7e1c5 100644
--- a/src/zenhttp/servers/httpplugin.cpp
+++ b/src/zenhttp/servers/httpplugin.cpp
@@ -2,6 +2,8 @@
 
 #include <zenhttp/httpplugin.h>
 
+#include "httptracer.h"
+
 #if ZEN_WITH_PLUGINS
 
 #	include "httpparser.h"
@@ -10,6 +12,7 @@
 #	include <zencore/filesystem.h>
 #	include <zencore/fmtutils.h>
 #	include <zencore/logging.h>
+#	include <zencore/memory/llm.h>
 #	include <zencore/scopeguard.h>
 #	include <zencore/session.h>
 #	include <zencore/thread.h>
@@ -25,14 +28,6 @@
 #		include <conio.h>
 #	endif
 
-#	define PLUGIN_VERBOSE_TRACE 1
-
-#	if PLUGIN_VERBOSE_TRACE
-#		define ZEN_TRACE_VERBOSE ZEN_TRACE
-#	else
-#		define ZEN_TRACE_VERBOSE(fmtstr, ...)
-#	endif
-
 namespace zen {
 
 struct HttpPluginServerImpl;
@@ -40,6 +35,14 @@ struct HttpPluginResponse;
 
 using namespace std::literals;
 
+const FLLMTag&
+GetHttppluginTag()
+{
+	static FLLMTag _("httpplugin");
+
+	return _;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 struct HttpPluginConnectionHandler : public TransportServerConnection, public HttpRequestParserCallbacks, RefCounted
@@ -103,8 +106,6 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer
 
 	HttpService* RouteRequest(std::string_view Url);
 
-	void WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload);
-
 	struct ServiceEntry
 	{
 		std::string	 ServiceUrlPath;
@@ -119,8 +120,8 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer
 	bool							  m_IsRequestLoggingEnabled = false;
 	LoggerRef						  m_RequestLog;
 	std::atomic_uint32_t			  m_ConnectionIdCounter{0};
-	std::filesystem::path			  m_DataDir;	 // Application data directory
-	std::filesystem::path			  m_PayloadDir;	 // Request debugging payload directory
+
+	HttpServerTracer m_RequestTracer;
 
 	// TransportServer
 
@@ -191,6 +192,7 @@ private:
 void
 HttpPluginResponse::InitializeForPayload(uint16_t ResponseCode, std::span<IoBuffer> BlobList)
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
 	ZEN_TRACE_CPU("http_plugin::InitializeForPayload");
 
 	m_ResponseCode			  = ResponseCode;
@@ -232,6 +234,8 @@ HttpPluginResponse::InitializeForPayload(uint16_t ResponseCode, std::span<IoBuff
 std::string_view
 HttpPluginResponse::GetHeaders()
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	if (m_Headers.Size() == 0)
 	{
 		m_Headers << "HTTP/1.1 " << ResponseCode() << " " << ReasonStringForHttpResultCode(ResponseCode()) << "\r\n"
@@ -266,6 +270,8 @@ HttpPluginConnectionHandler::~HttpPluginConnectionHandler()
 void
 HttpPluginConnectionHandler::Initialize(TransportConnection* Transport, HttpPluginServerImpl& Server, uint32_t ConnectionId)
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	m_TransportConnection = Transport;
 	m_Server			  = &Server;
 	m_ConnectionId		  = ConnectionId;
@@ -298,6 +304,8 @@ HttpPluginConnectionHandler::Release() const
 void
 HttpPluginConnectionHandler::OnBytesRead(const void* Buffer, size_t AvailableBytes)
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	ZEN_ASSERT(m_Server);
 
 	ZEN_LOG_TRACE(m_Server->m_RequestLog, "connection #{} OnBytesRead: {}", m_ConnectionId, AvailableBytes);
@@ -325,6 +333,8 @@ HttpPluginConnectionHandler::OnBytesRead(const void* Buffer, size_t AvailableByt
 void
 HttpPluginConnectionHandler::HandleRequest()
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	ZEN_ASSERT(m_Server);
 
 	const uint32_t RequestNumber = m_RequestCounter.fetch_add(1);
@@ -376,8 +386,8 @@ HttpPluginConnectionHandler::HandleRequest()
 						  ToString(Request.RequestContentType()),
 						  ToString(Request.AcceptContentType()));
 
-			m_Server->WriteDebugPayload(fmt::format("request_{}_{}.bin", m_ConnectionId, RequestNumber),
-										std::vector<IoBuffer>{Request.ReadPayload()});
+			m_Server->m_RequestTracer.WriteDebugPayload(fmt::format("request_{}_{}.bin", m_ConnectionId, RequestNumber),
+														std::vector<IoBuffer>{Request.ReadPayload()});
 		}
 
 		if (!HandlePackageOffers(*Service, Request, m_PackageHandler))
@@ -386,7 +396,15 @@ HttpPluginConnectionHandler::HandleRequest()
 			{
 				Service->HandleRequest(Request);
 			}
-			catch (std::system_error& SystemError)
+			catch (const AssertException& AssertEx)
+			{
+				// Drop any partially formatted response
+				Request.m_Response.reset();
+
+				ZEN_ERROR("Caught assert exception while handling request: {}", AssertEx.FullDescription());
+				Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, AssertEx.FullDescription());
+			}
+			catch (const std::system_error& SystemError)
 			{
 				// Drop any partially formatted response
 				Request.m_Response.reset();
@@ -397,23 +415,25 @@ HttpPluginConnectionHandler::HandleRequest()
 				}
 				else
 				{
-					ZEN_ERROR("Caught system error exception while handling request: {}", SystemError.what());
+					ZEN_WARN("Caught system error exception while handling request: {}. ({})",
+							 SystemError.what(),
+							 SystemError.code().value());
 					Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, SystemError.what());
 				}
 			}
-			catch (std::bad_alloc& BadAlloc)
+			catch (const std::bad_alloc& BadAlloc)
 			{
 				// Drop any partially formatted response
 				Request.m_Response.reset();
 
 				Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, BadAlloc.what());
 			}
-			catch (std::exception& ex)
+			catch (const std::exception& ex)
 			{
 				// Drop any partially formatted response
 				Request.m_Response.reset();
 
-				ZEN_ERROR("Caught exception while handling request: {}", ex.what());
+				ZEN_WARN("Caught exception while handling request: {}", ex.what());
 				Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, ex.what());
 			}
 		}
@@ -442,7 +462,8 @@ HttpPluginConnectionHandler::HandleRequest()
 
 			if (m_Server->m_RequestLog.ShouldLog(logging::level::Trace))
 			{
-				m_Server->WriteDebugPayload(fmt::format("response_{}_{}.bin", m_ConnectionId, RequestNumber), ResponseBuffers);
+				m_Server->m_RequestTracer.WriteDebugPayload(fmt::format("response_{}_{}.bin", m_ConnectionId, RequestNumber),
+															ResponseBuffers);
 			}
 
 			for (const IoBuffer& Buffer : ResponseBuffers)
@@ -523,6 +544,7 @@ HttpPluginConnectionHandler::HandleRequest()
 void
 HttpPluginConnectionHandler::TerminateConnection()
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
 	ZEN_ASSERT(m_TransportConnection);
 	m_TransportConnection->CloseConnection();
 }
@@ -533,6 +555,8 @@ HttpPluginServerRequest::HttpPluginServerRequest(HttpRequestParser& Request, Htt
 : m_Request(Request)
 , m_PayloadBuffer(std::move(PayloadBuffer))
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	const int PrefixLength = Service.UriPrefixLength();
 
 	std::string_view Uri = Request.Url();
@@ -613,6 +637,7 @@ void
 HttpPluginServerRequest::WriteResponse(HttpResponseCode ResponseCode)
 {
 	ZEN_ASSERT(!m_Response);
+	ZEN_MEMSCOPE(GetHttppluginTag());
 
 	m_Response.reset(new HttpPluginResponse(HttpContentType::kBinary));
 	std::array<IoBuffer, 0> Empty;
@@ -624,6 +649,7 @@ void
 HttpPluginServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs)
 {
 	ZEN_ASSERT(!m_Response);
+	ZEN_MEMSCOPE(GetHttppluginTag());
 
 	m_Response.reset(new HttpPluginResponse(ContentType));
 	m_Response->InitializeForPayload((uint16_t)ResponseCode, Blobs);
@@ -633,6 +659,8 @@ void
 HttpPluginServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::u8string_view ResponseString)
 {
 	ZEN_ASSERT(!m_Response);
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	m_Response.reset(new HttpPluginResponse(ContentType));
 
 	IoBuffer				MessageBuffer(IoBuffer::Wrap, ResponseString.data(), ResponseString.size());
@@ -645,6 +673,7 @@ void
 HttpPluginServerRequest::WriteResponseAsync(std::function<void(HttpServerRequest&)>&& ContinuationHandler)
 {
 	ZEN_ASSERT(!m_Response);
+	ZEN_MEMSCOPE(GetHttppluginTag());
 
 	// Not one bit async, innit
 	ContinuationHandler(*this);
@@ -669,6 +698,7 @@ HttpPluginServerImpl::~HttpPluginServerImpl()
 TransportServerConnection*
 HttpPluginServerImpl::CreateConnectionHandler(TransportConnection* Connection)
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
 	HttpPluginConnectionHandler* Handler{new HttpPluginConnectionHandler()};
 	const uint32_t				 ConnectionId = m_ConnectionIdCounter.fetch_add(1);
 	Handler->Initialize(Connection, *this, ConnectionId);
@@ -678,10 +708,10 @@ HttpPluginServerImpl::CreateConnectionHandler(TransportConnection* Connection)
 int
 HttpPluginServerImpl::Initialize(int BasePort, std::filesystem::path DataDir)
 {
-	m_DataDir	 = DataDir;
-	m_PayloadDir = DataDir / "debug" / GetSessionIdString();
+	ZEN_TRACE_CPU("HttpPluginServerImpl::Initialize");
 
-	ZEN_INFO("any debug payloads will be written to '{}'", m_PayloadDir);
+	ZEN_MEMSCOPE(GetHttppluginTag());
+	m_RequestTracer.Initialize(DataDir);
 
 	try
 	{
@@ -693,13 +723,13 @@ HttpPluginServerImpl::Initialize(int BasePort, std::filesystem::path DataDir)
 			{
 				Plugin->Initialize(this);
 			}
-			catch (std::exception& Ex)
+			catch (const std::exception& Ex)
 			{
 				ZEN_WARN("exception caught during plugin initialization: {}", Ex.what());
 			}
 		}
 	}
-	catch (std::exception& ex)
+	catch (const std::exception& ex)
 	{
 		ZEN_WARN("Caught exception starting http plugin server: {}", ex.what());
 	}
@@ -715,6 +745,8 @@ HttpPluginServerImpl::Close()
 	if (!m_IsInitialized)
 		return;
 
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	try
 	{
 		RwLock::ExclusiveLockScope _(m_Lock);
@@ -725,7 +757,7 @@ HttpPluginServerImpl::Close()
 			{
 				Plugin->Shutdown();
 			}
-			catch (std::exception& Ex)
+			catch (const std::exception& Ex)
 			{
 				ZEN_WARN("exception caught during plugin shutdown: {}", Ex.what());
 			}
@@ -735,7 +767,7 @@ HttpPluginServerImpl::Close()
 
 		m_Plugins.clear();
 	}
-	catch (std::exception& ex)
+	catch (const std::exception& ex)
 	{
 		ZEN_WARN("Caught exception stopping http plugin server: {}", ex.what());
 	}
@@ -746,6 +778,8 @@ HttpPluginServerImpl::Close()
 void
 HttpPluginServerImpl::Run(bool IsInteractive)
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	const bool TestMode = !IsInteractive;
 
 	int WaitTimeout = -1;
@@ -796,6 +830,8 @@ HttpPluginServerImpl::RequestExit()
 void
 HttpPluginServerImpl::AddPlugin(Ref<TransportPlugin> Plugin)
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	RwLock::ExclusiveLockScope _(m_Lock);
 	m_Plugins.emplace_back(std::move(Plugin));
 }
@@ -803,6 +839,8 @@ HttpPluginServerImpl::AddPlugin(Ref<TransportPlugin> Plugin)
 void
 HttpPluginServerImpl::RemovePlugin(Ref<TransportPlugin> Plugin)
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	RwLock::ExclusiveLockScope _(m_Lock);
 	auto					   It = std::find(begin(m_Plugins), end(m_Plugins), Plugin);
 	if (It != m_Plugins.end())
@@ -814,6 +852,8 @@ HttpPluginServerImpl::RemovePlugin(Ref<TransportPlugin> Plugin)
 void
 HttpPluginServerImpl::RegisterService(HttpService& Service)
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	std::string_view UrlPath(Service.BaseUri());
 	Service.SetUriPrefixLength(UrlPath.size());
 
@@ -829,6 +869,8 @@ HttpPluginServerImpl::RegisterService(HttpService& Service)
 HttpService*
 HttpPluginServerImpl::RouteRequest(std::string_view Url)
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
+
 	RwLock::SharedLockScope _(m_Lock);
 
 	HttpService*		   CandidateService	  = nullptr;
@@ -848,23 +890,6 @@ HttpPluginServerImpl::RouteRequest(std::string_view Url)
 	return CandidateService;
 }
 
-void
-HttpPluginServerImpl::WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload)
-{
-	uint64_t					 PayloadSize = 0;
-	std::vector<const IoBuffer*> Buffers;
-	for (auto& Io : Payload)
-	{
-		Buffers.push_back(&Io);
-		PayloadSize += Io.GetSize();
-	}
-
-	if (PayloadSize)
-	{
-		WriteFile(m_PayloadDir / Filename, Buffers.data(), Buffers.size());
-	}
-}
-
 //////////////////////////////////////////////////////////////////////////
 
 struct HttpPluginServerImpl;
@@ -872,6 +897,7 @@ struct HttpPluginServerImpl;
 Ref<HttpPluginServer>
 CreateHttpPluginServer()
 {
+	ZEN_MEMSCOPE(GetHttppluginTag());
 	return Ref<HttpPluginServer>(new HttpPluginServerImpl);
 }
 
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index 5cd273c40..95d83911d 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -9,11 +9,14 @@
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
+#include <zencore/memory/llm.h>
 #include <zencore/scopeguard.h>
 #include <zencore/string.h>
 #include <zencore/timer.h>
 #include <zencore/trace.h>
-#include <zenutil/packageformat.h>
+#include <zenhttp/packageformat.h>
+
+#include <EASTL/fixed_vector.h>
 
 #if ZEN_WITH_HTTPSYS
 #	define _WINSOCKAPI_
@@ -25,6 +28,14 @@
 
 namespace zen {
 
+const FLLMTag&
+GetHttpsysTag()
+{
+	static FLLMTag HttpsysTag("httpsys");
+
+	return HttpsysTag;
+}
+
 /**
  * @brief Windows implementation of HTTP server based on http.sys
  *
@@ -372,14 +383,14 @@ public:
 	void						   SuppressResponseBody();	// typically used for HEAD requests
 
 private:
-	std::vector<HTTP_DATA_CHUNK> m_HttpDataChunks;
-	uint64_t					 m_TotalDataSize	   = 0;	 // Sum of all chunk sizes
-	uint16_t					 m_ResponseCode		   = 0;
-	uint32_t					 m_NextDataChunkOffset = 0;	 // Cursor used for very large chunk lists
-	uint32_t					 m_RemainingChunkCount = 0;	 // Backlog for multi-call sends
-	bool						 m_IsInitialResponse   = true;
-	HttpContentType				 m_ContentType		   = HttpContentType::kBinary;
-	std::vector<IoBuffer>		 m_DataBuffers;
+	eastl::fixed_vector<HTTP_DATA_CHUNK, 16> m_HttpDataChunks;
+	uint64_t								 m_TotalDataSize	   = 0;	 // Sum of all chunk sizes
+	uint16_t								 m_ResponseCode		   = 0;
+	uint32_t								 m_NextDataChunkOffset = 0;	 // Cursor used for very large chunk lists
+	uint32_t								 m_RemainingChunkCount = 0;	 // Backlog for multi-call sends
+	bool									 m_IsInitialResponse   = true;
+	HttpContentType							 m_ContentType		   = HttpContentType::kBinary;
+	eastl::fixed_vector<IoBuffer, 16>		 m_DataBuffers;
 
 	void InitializeForPayload(uint16_t ResponseCode, std::span<IoBuffer> Blobs);
 };
@@ -524,7 +535,14 @@ HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfB
 
 	if (IoResult != NO_ERROR)
 	{
-		ZEN_WARN("response aborted due to error: '{}'", GetSystemErrorAsString(IoResult));
+		ZEN_WARN("response '{}' ({}) aborted after transfering '{}', {} out of {} bytes, reason: {} ({})",
+				 ReasonStringForHttpResultCode(m_ResponseCode),
+				 m_ResponseCode,
+				 ToString(m_ContentType),
+				 NumberOfBytesTransferred,
+				 m_TotalDataSize,
+				 GetSystemErrorAsString(IoResult),
+				 IoResult);
 
 		// if one transmit failed there's really no need to go on
 		return nullptr;
@@ -673,7 +691,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
 		);
 	}
 
-	auto EmitReponseDetails = [&](StringBuilderBase& ResponseDetails) -> void {
+	auto EmitResponseDetails = [&](StringBuilderBase& ResponseDetails) -> void {
 		for (int i = 0; i < ThisRequestChunkCount; ++i)
 		{
 			const HTTP_DATA_CHUNK Chunk = m_HttpDataChunks[ThisRequestChunkOffset + i];
@@ -756,7 +774,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
 		// Emit diagnostics
 
 		ExtendableStringBuilder<256> ResponseDetails;
-		EmitReponseDetails(ResponseDetails);
+		EmitResponseDetails(ResponseDetails);
 
 		ZEN_WARN("failed to send HTTP response (error {}: '{}'), request URL: '{}', ({}.{}) response: {}",
 				 SendResult,
@@ -817,7 +835,7 @@ HttpAsyncWorkRequest::IssueRequest(std::error_code& ErrorCode)
 	ZEN_TRACE_CPU("httpsys::AsyncWork::IssueRequest");
 	ErrorCode.clear();
 
-	Transaction().Server().WorkPool().ScheduleWork(m_WorkItem);
+	Transaction().Server().WorkPool().ScheduleWork(m_WorkItem, WorkerThreadPool::EMode::EnableBacklog);
 }
 
 HttpSysRequestHandler*
@@ -836,6 +854,8 @@ HttpAsyncWorkRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTr
 void
 HttpAsyncWorkRequest::AsyncWorkItem::Execute()
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	ZEN_TRACE_CPU("httpsys::async_execute");
 
 	try
@@ -873,10 +893,15 @@ HttpAsyncWorkRequest::AsyncWorkItem::Execute()
 				new HttpMessageResponseRequest(Tx, 500, "Response generated but no request handler scheduled"sv));
 		}
 	}
-	catch (std::exception& Ex)
+	catch (const AssertException& AssertEx)
 	{
 		return (void)Tx.IssueNextRequest(
-			new HttpMessageResponseRequest(Tx, 500, fmt::format("Exception thrown in async work: '{}'", Ex.what())));
+			new HttpMessageResponseRequest(Tx, 500, fmt::format("Assert thrown in async work: '{}", AssertEx.FullDescription())));
+	}
+	catch (const std::exception& Ex)
+	{
+		return (void)Tx.IssueNextRequest(
+			new HttpMessageResponseRequest(Tx, 500, fmt::format("Exception thrown in async work: {}", Ex.what())));
 	}
 }
 
@@ -896,6 +921,8 @@ HttpSysServer::HttpSysServer(const HttpSysConfig& InConfig)
 , m_IsAsyncResponseEnabled(InConfig.IsAsyncResponseEnabled)
 , m_InitialConfig(InConfig)
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	// Initialize thread pool
 
 	int MinThreadCount;
@@ -971,6 +998,8 @@ HttpSysServer::Close()
 int
 HttpSysServer::InitializeServer(int BasePort)
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	using namespace std::literals;
 
 	WideStringBuilder<64> WildcardUrlPath;
@@ -1215,6 +1244,8 @@ HttpSysServer::Cleanup()
 WorkerThreadPool&
 HttpSysServer::WorkPool()
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	if (!m_AsyncWorkPool)
 	{
 		RwLock::ExclusiveLockScope _(m_AsyncWorkPoolInitLock);
@@ -1299,6 +1330,8 @@ HttpSysServer::IssueNewRequestMaybe()
 		return;
 	}
 
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	std::unique_ptr<HttpSysTransaction> Request = std::make_unique<HttpSysTransaction>(*this);
 
 	std::error_code ErrorCode;
@@ -1322,6 +1355,8 @@ HttpSysServer::IssueNewRequestMaybe()
 void
 HttpSysServer::RegisterService(const char* UrlPath, HttpService& Service)
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	if (UrlPath[0] == '/')
 	{
 		++UrlPath;
@@ -1483,11 +1518,15 @@ HttpSysTransaction::IssueNextRequest(HttpSysRequestHandler* NewCompletionHandler
 			return true;
 		}
 
-		ZEN_WARN("IssueRequest() failed: '{}'", ErrorCode.message());
+		ZEN_WARN("IssueRequest() failed: {}", ErrorCode.message());
+	}
+	catch (const AssertException& AssertEx)
+	{
+		ZEN_ERROR("Assert thrown in IssueNextRequest(): {}", AssertEx.FullDescription());
 	}
-	catch (std::exception& Ex)
+	catch (const std::exception& Ex)
 	{
-		ZEN_ERROR("exception caught in IssueNextRequest(): '{}'", Ex.what());
+		ZEN_ERROR("exception caught in IssueNextRequest(): {}", Ex.what());
 	}
 
 	// something went wrong, no request is pending
@@ -1659,7 +1698,7 @@ HttpSysServerRequest::ParseSessionId() const
 		{
 			if (Header.RawValueLength == Oid::StringLength)
 			{
-				return Oid::FromHexString({Header.pRawValue, Header.RawValueLength});
+				return Oid::TryFromHexString({Header.pRawValue, Header.RawValueLength});
 			}
 		}
 	}
@@ -1698,6 +1737,8 @@ HttpSysServerRequest::ReadPayload()
 void
 HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode)
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	ZEN_ASSERT(IsHandled() == false);
 
 	auto Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)ResponseCode);
@@ -1715,6 +1756,8 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode)
 void
 HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs)
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	ZEN_ASSERT(IsHandled() == false);
 
 	auto Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)ResponseCode, ContentType, Blobs);
@@ -1732,6 +1775,8 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentTy
 void
 HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::u8string_view ResponseString)
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	ZEN_ASSERT(IsHandled() == false);
 
 	auto Response =
@@ -1750,6 +1795,8 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentTy
 void
 HttpSysServerRequest::WriteResponseAsync(std::function<void(HttpServerRequest&)>&& ContinuationHandler)
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	if (m_HttpTx.Server().IsAsyncResponseEnabled())
 	{
 		m_NextCompletionHandler = new HttpAsyncWorkRequest(m_HttpTx, std::move(ContinuationHandler));
@@ -1826,7 +1873,17 @@ InitialRequestHandler::IssueRequest(std::error_code& ErrorCode)
 
 		ErrorCode = MakeErrorCode(HttpApiResult);
 
-		ZEN_WARN("HttpReceiveHttpRequest failed, error: '{}'", ErrorCode.message());
+		if (IsInitialRequest())
+		{
+			ZEN_WARN("initial HttpReceiveHttpRequest failed, error: {}", ErrorCode.message());
+		}
+		else
+		{
+			ZEN_WARN("HttpReceiveHttpRequest (offset: {}, content-length: {}) failed, error: {}",
+					 m_CurrentPayloadOffset,
+					 m_PayloadBuffer.GetSize(),
+					 ErrorCode.message());
+		}
 
 		return;
 	}
@@ -1837,6 +1894,8 @@ InitialRequestHandler::IssueRequest(std::error_code& ErrorCode)
 HttpSysRequestHandler*
 InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred)
 {
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	auto _ = MakeGuard([&] { m_IsInitialRequest = false; });
 
 	switch (IoResult)
@@ -1985,23 +2044,28 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT
 		// Unable to route
 		return new HttpMessageResponseRequest(Transaction(), 404, "No suitable route found"sv);
 	}
-	catch (std::system_error& SystemError)
+	catch (const AssertException& AssertEx)
+	{
+		ZEN_ERROR("Caught assert exception while handling request: {}", AssertEx.FullDescription());
+		return new HttpMessageResponseRequest(Transaction(), (uint16_t)HttpResponseCode::InternalServerError, AssertEx.FullDescription());
+	}
+	catch (const std::system_error& SystemError)
 	{
 		if (IsOOM(SystemError.code()) || IsOOD(SystemError.code()))
 		{
 			return new HttpMessageResponseRequest(Transaction(), (uint16_t)HttpResponseCode::InsufficientStorage, SystemError.what());
 		}
 
-		ZEN_ERROR("Caught system error exception while handling request: {}", SystemError.what());
+		ZEN_WARN("Caught system error exception while handling request: {}. ({})", SystemError.what(), SystemError.code().value());
 		return new HttpMessageResponseRequest(Transaction(), (uint16_t)HttpResponseCode::InternalServerError, SystemError.what());
 	}
-	catch (std::bad_alloc& BadAlloc)
+	catch (const std::bad_alloc& BadAlloc)
 	{
 		return new HttpMessageResponseRequest(Transaction(), (uint16_t)HttpResponseCode::InsufficientStorage, BadAlloc.what());
 	}
-	catch (std::exception& ex)
+	catch (const std::exception& ex)
 	{
-		ZEN_ERROR("Caught exception while handling request: '{}'", ex.what());
+		ZEN_WARN("Caught exception while handling request: '{}'", ex.what());
 		return new HttpMessageResponseRequest(Transaction(), (uint16_t)HttpResponseCode::InternalServerError, ex.what());
 	}
 }
@@ -2014,6 +2078,8 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT
 int
 HttpSysServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_TRACE_CPU("HttpSysServer::Initialize");
+
 	ZEN_UNUSED(DataDir);
 	if (int EffectivePort = InitializeServer(BasePort))
 	{
@@ -2042,6 +2108,9 @@ HttpSysServer::RegisterService(HttpService& Service)
 Ref<HttpServer>
 CreateHttpSysServer(HttpSysConfig Config)
 {
+	ZEN_TRACE_CPU("CreateHttpSysServer");
+	ZEN_MEMSCOPE(GetHttpsysTag());
+
 	return Ref<HttpServer>(new HttpSysServer(Config));
 }
 
diff --git a/src/zenhttp/servers/httptracer.cpp b/src/zenhttp/servers/httptracer.cpp
new file mode 100644
index 000000000..483307fb1
--- /dev/null
+++ b/src/zenhttp/servers/httptracer.cpp
@@ -0,0 +1,37 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "httptracer.h"
+
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/session.h>
+
+namespace zen {
+
+void
+HttpServerTracer::Initialize(std::filesystem::path DataDir)
+{
+	m_DataDir	 = DataDir;
+	m_PayloadDir = DataDir / "debug" / GetSessionIdString();
+
+	ZEN_INFO("any debug payloads will be written to '{}'", m_PayloadDir);
+}
+
+void
+HttpServerTracer::WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload)
+{
+	uint64_t					 PayloadSize = 0;
+	std::vector<const IoBuffer*> Buffers;
+	for (auto& Io : Payload)
+	{
+		Buffers.push_back(&Io);
+		PayloadSize += Io.GetSize();
+	}
+
+	if (PayloadSize)
+	{
+		WriteFile(m_PayloadDir / Filename, Buffers.data(), Buffers.size());
+	}
+}
+
+}  // namespace zen
diff --git a/src/zenhttp/servers/httptracer.h b/src/zenhttp/servers/httptracer.h
new file mode 100644
index 000000000..da72c79c9
--- /dev/null
+++ b/src/zenhttp/servers/httptracer.h
@@ -0,0 +1,26 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenhttp/httpserver.h>
+
+#pragma once
+
+namespace zen {
+
+/** Helper class for HTTP server implementations
+
+	Provides some common functionality which can be used across all server
+	implementations. These could be in the root class but I think it's nicer
+	to hide the implementation details from client code
+ */
+class HttpServerTracer
+{
+public:
+	void Initialize(std::filesystem::path DataDir);
+	void WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload);
+
+private:
+	std::filesystem::path m_DataDir;	 // Application data directory
+	std::filesystem::path m_PayloadDir;	 // Request debugging payload directory
+};
+
+}  // namespace zen