HTTP improvements (#803)

- Add GetTotalBytesReceived/GetTotalBytesSent to HttpServer with implementations in ASIO and http.sys backends
- Add ExpectedErrorCodes to HttpClientSettings to suppress warn/info logs for anticipated HTTP error codes
- Also fixes minor issues in `CprHttpClient::Download`

8 files changed, 99 insertions, 19 deletions

diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index 226a94050..f657869dc 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -9,6 +9,7 @@
 #include <zencore/string.h>
 #include <zencore/testing.h>
 #include <zencore/timer.h>
+#include <zencore/trace.h>
 
 #include <thread>
 
@@ -745,6 +746,8 @@ CreateProcElevated(const std::filesystem::path& Executable, std::string_view Com
 CreateProcResult
 CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options)
 {
+	ZEN_TRACE_CPU("CreateProc");
+
 #if ZEN_PLATFORM_WINDOWS
 	if (Options.Flags & CreateProcOptions::Flag_Unelevated)
 	{
@@ -776,6 +779,17 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine
 			ZEN_UNUSED(Result);
 		}
 
+		if (!Options.StdoutFile.empty())
+		{
+			int Fd = open(Options.StdoutFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
+			if (Fd >= 0)
+			{
+				dup2(Fd, STDOUT_FILENO);
+				dup2(Fd, STDERR_FILENO);
+				close(Fd);
+			}
+		}
+
 		if (execv(Executable.c_str(), ArgV.data()) < 0)
 		{
 			ThrowLastError("Failed to exec() a new process image");
diff --git a/src/zencore/windows.cpp b/src/zencore/windows.cpp
index d02fcd35e..87f854b90 100644
--- a/src/zencore/windows.cpp
+++ b/src/zencore/windows.cpp
@@ -12,14 +12,12 @@ namespace zen::windows {
 bool
 IsRunningOnWine()
 {
-	HMODULE NtDll = GetModuleHandleA("ntdll.dll");
+	static bool s_Result = [] {
+		HMODULE NtDll = GetModuleHandleA("ntdll.dll");
+		return NtDll && !!GetProcAddress(NtDll, "wine_get_version");
+	}();
 
-	if (NtDll)
-	{
-		return !!GetProcAddress(NtDll, "wine_get_version");
-	}
-
-	return false;
+	return s_Result;
 }
 
 FileMapping::FileMapping(_In_ FileMapping& orig)
diff --git a/src/zenhttp/clients/httpclientcpr.cpp b/src/zenhttp/clients/httpclientcpr.cpp
index 90dcfacbb..14e40b02a 100644
--- a/src/zenhttp/clients/httpclientcpr.cpp
+++ b/src/zenhttp/clients/httpclientcpr.cpp
@@ -12,6 +12,7 @@
 #include <zencore/session.h>
 #include <zencore/stream.h>
 #include <zenhttp/packageformat.h>
+#include <algorithm>
 
 namespace zen {
 
@@ -164,6 +165,18 @@ CprHttpClient::CprHttpClient(std::string_view		   BaseUri,
 {
 }
 
+bool
+CprHttpClient::ShouldLogErrorCode(HttpResponseCode ResponseCode) const
+{
+	if (m_CheckIfAbortFunction && m_CheckIfAbortFunction())
+	{
+		// Quiet
+		return false;
+	}
+	const auto& Expected = m_ConnectionSettings.ExpectedErrorCodes;
+	return std::find(Expected.begin(), Expected.end(), ResponseCode) == Expected.end();
+}
+
 CprHttpClient::~CprHttpClient()
 {
 	ZEN_TRACE_CPU("CprHttpClient::~CprHttpClient");
@@ -193,11 +206,9 @@ CprHttpClient::ResponseWithPayload(std::string_view										  SessionId,
 		ResponseBuffer.SetContentType(ContentType);
 	}
 
-	const bool Quiet = m_CheckIfAbortFunction && m_CheckIfAbortFunction();
-
-	if (!Quiet)
+	if (!IsHttpSuccessCode(WorkResponseCode) && WorkResponseCode != HttpResponseCode::NotFound)
 	{
-		if (!IsHttpSuccessCode(WorkResponseCode) && WorkResponseCode != HttpResponseCode::NotFound)
+		if (ShouldLogErrorCode(WorkResponseCode))
 		{
 			ZEN_WARN("HttpClient request failed (session: {}): {}", SessionId, HttpResponse);
 		}
@@ -371,8 +382,7 @@ CprHttpClient::DoWithRetry(std::string_view								SessionId,
 		}
 		Sleep(100 * (Attempt + 1));
 		Attempt++;
-		const bool Quiet = m_CheckIfAbortFunction && m_CheckIfAbortFunction();
-		if (!Quiet)
+		if (ShouldLogErrorCode(HttpResponseCode(Result.status_code)))
 		{
 			ZEN_INFO("{} Attempt {}/{}",
 					 CommonResponse(SessionId, std::move(Result), {}).ErrorMessage("Retry"),
@@ -410,8 +420,7 @@ CprHttpClient::DoWithRetry(std::string_view							 SessionId,
 		}
 		Sleep(100 * (Attempt + 1));
 		Attempt++;
-		const bool Quiet = m_CheckIfAbortFunction && m_CheckIfAbortFunction();
-		if (!Quiet)
+		if (ShouldLogErrorCode(HttpResponseCode(Result.status_code)))
 		{
 			ZEN_INFO("{} Attempt {}/{}",
 					 CommonResponse(SessionId, std::move(Result), {}).ErrorMessage("Retry"),
@@ -646,7 +655,7 @@ CprHttpClient::TransactPackage(std::string_view Url, CbPackage Package, const Ke
 		ResponseBuffer.SetContentType(ContentType);
 	}
 
-	return {.StatusCode = HttpResponseCode(FilterResponse.status_code), .ResponsePayload = ResponseBuffer};
+	return {.StatusCode = HttpResponseCode(FilterResponse.status_code), .ResponsePayload = std::move(ResponseBuffer)};
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -929,6 +938,13 @@ CprHttpClient::Download(std::string_view Url, const std::filesystem::path& TempF
 	cpr::Response Response = DoWithRetry(
 		m_SessionId,
 		[&]() {
+			// Reset state from any previous attempt
+			PayloadString.clear();
+			PayloadFile.reset();
+			BoundaryParser.Boundaries.clear();
+			ContentType			 = HttpContentType::kUnknownContentType;
+			IsMultiRangeResponse = false;
+
 			auto DownloadCallback = [&](std::string data, intptr_t) {
 				if (m_CheckIfAbortFunction && m_CheckIfAbortFunction())
 				{
@@ -969,7 +985,7 @@ CprHttpClient::Download(std::string_view Url, const std::filesystem::path& TempF
 					if (RangeStartPos != std::string::npos)
 					{
 						RangeStartPos++;
-						while (RangeValue[RangeStartPos] == ' ')
+						while (RangeStartPos < RangeValue.length() && RangeValue[RangeStartPos] == ' ')
 						{
 							RangeStartPos++;
 						}
@@ -991,7 +1007,7 @@ CprHttpClient::Download(std::string_view Url, const std::filesystem::path& TempF
 								std::optional<size_t> RequestedRangeEnd	  = ParseInt<size_t>(RangeString.substr(RangeSplitPos + 1));
 								if (RequestedRangeStart.has_value() && RequestedRangeEnd.has_value())
 								{
-									RequestedContentLength += RequestedRangeEnd.value() - 1;
+									RequestedContentLength += RequestedRangeEnd.value() - RequestedRangeStart.value() + 1;
 								}
 							}
 							RangeStartPos = RangeEnd;
diff --git a/src/zenhttp/clients/httpclientcpr.h b/src/zenhttp/clients/httpclientcpr.h
index cf2d3bd14..752d91add 100644
--- a/src/zenhttp/clients/httpclientcpr.h
+++ b/src/zenhttp/clients/httpclientcpr.h
@@ -155,6 +155,7 @@ private:
 		std::function<cpr::Response()>&&			 Func,
 		std::function<bool(cpr::Response& Result)>&& Validate = [](cpr::Response&) { return true; });
 
+	bool ShouldLogErrorCode(HttpResponseCode ResponseCode) const;
 	bool ValidatePayload(cpr::Response& Response, std::unique_ptr<detail::TempPayloadFile>& PayloadFile);
 
 	HttpClient::Response CommonResponse(std::string_view									   SessionId,
diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h
index 53be36b9a..d87082d10 100644
--- a/src/zenhttp/include/zenhttp/httpclient.h
+++ b/src/zenhttp/include/zenhttp/httpclient.h
@@ -13,6 +13,7 @@
 #include <functional>
 #include <optional>
 #include <unordered_map>
+#include <vector>
 
 namespace zen {
 
@@ -58,6 +59,10 @@ struct HttpClientSettings
 	Oid													  SessionId					  = Oid::Zero;
 	bool												  Verbose					  = false;
 	uint64_t											  MaximumInMemoryDownloadSize = 1024u * 1024u;
+
+	/// HTTP status codes that are expected and should not be logged as warnings.
+	/// 404 is always treated as expected regardless of this list.
+	std::vector<HttpResponseCode> ExpectedErrorCodes;
 };
 
 class HttpClientError : public std::runtime_error
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
index fee932daa..62c080a7b 100644
--- a/src/zenhttp/include/zenhttp/httpserver.h
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -230,6 +230,10 @@ public:
 	 */
 	std::string_view GetExternalHost() const { return m_ExternalHost; }
 
+	/** Returns total bytes received and sent across all connections since server start. */
+	virtual uint64_t GetTotalBytesReceived() const { return 0; }
+	virtual uint64_t GetTotalBytesSent() const { return 0; }
+
 private:
 	std::vector<HttpService*> m_KnownServices;
 	int						  m_EffectivePort = 0;
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index 8c2dcd116..c4d9ee777 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -528,6 +528,9 @@ public:
 
 	RwLock					  m_Lock;
 	std::vector<ServiceEntry> m_UriHandlers;
+
+	std::atomic<uint64_t> m_TotalBytesReceived{0};
+	std::atomic<uint64_t> m_TotalBytesSent{0};
 };
 
 /**
@@ -1043,6 +1046,8 @@ HttpServerConnection::OnDataReceived(const asio::error_code& Ec, [[maybe_unused]
 		}
 	}
 
+	m_Server.m_TotalBytesReceived.fetch_add(ByteCount, std::memory_order_relaxed);
+
 	ZEN_TRACE_VERBOSE("on data received, connection: {}, request: {}, thread: {}, bytes: {}",
 					  m_ConnectionId,
 					  m_RequestCounter.load(std::memory_order_relaxed),
@@ -1096,6 +1101,8 @@ HttpServerConnection::OnResponseDataSent(const asio::error_code&	  Ec,
 		return;
 	}
 
+	m_Server.m_TotalBytesSent.fetch_add(ByteCount, std::memory_order_relaxed);
+
 	ZEN_TRACE_VERBOSE("on data sent, connection: {}, request: {}, thread: {}, bytes: {}",
 					  m_ConnectionId,
 					  RequestNumber,
@@ -2053,6 +2060,8 @@ public:
 	virtual void		OnRequestExit() override;
 	virtual void		OnClose() override;
 	virtual std::string OnGetExternalHost() const override;
+	virtual uint64_t	GetTotalBytesReceived() const override;
+	virtual uint64_t	GetTotalBytesSent() const override;
 
 private:
 	Event			 m_ShutdownEvent;
@@ -2150,6 +2159,18 @@ HttpAsioServer::OnGetExternalHost() const
 	}
 }
 
+uint64_t
+HttpAsioServer::GetTotalBytesReceived() const
+{
+	return m_Impl->m_TotalBytesReceived.load(std::memory_order_relaxed);
+}
+
+uint64_t
+HttpAsioServer::GetTotalBytesSent() const
+{
+	return m_Impl->m_TotalBytesSent.load(std::memory_order_relaxed);
+}
+
 void
 HttpAsioServer::OnRun(bool IsInteractive)
 {
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index 23d57af57..a48f1d316 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -88,6 +88,8 @@ class HttpSysServerRequest;
 class HttpSysServer : public HttpServer
 {
 	friend class HttpSysTransaction;
+	friend class HttpMessageResponseRequest;
+	friend struct InitialRequestHandler;
 
 public:
 	explicit HttpSysServer(const HttpSysConfig& Config);
@@ -102,6 +104,8 @@ public:
 	virtual void		OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override;
 	virtual void		OnClose() override;
 	virtual std::string OnGetExternalHost() const override;
+	virtual uint64_t	GetTotalBytesReceived() const override;
+	virtual uint64_t	GetTotalBytesSent() const override;
 
 	WorkerThreadPool& WorkPool();
 
@@ -149,6 +153,9 @@ private:
 
 	RwLock							 m_RequestFilterLock;
 	std::atomic<IHttpRequestFilter*> m_HttpRequestFilter = nullptr;
+
+	std::atomic<uint64_t> m_TotalBytesReceived{0};
+	std::atomic<uint64_t> m_TotalBytesSent{0};
 };
 
 }  // namespace zen
@@ -591,7 +598,7 @@ HttpMessageResponseRequest::SuppressResponseBody()
 HttpSysRequestHandler*
 HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred)
 {
-	ZEN_UNUSED(NumberOfBytesTransferred);
+	Transaction().Server().m_TotalBytesSent.fetch_add(NumberOfBytesTransferred, std::memory_order_relaxed);
 
 	if (IoResult != NO_ERROR)
 	{
@@ -2123,6 +2130,8 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT
 			break;
 	}
 
+	Transaction().Server().m_TotalBytesReceived.fetch_add(NumberOfBytesTransferred, std::memory_order_relaxed);
+
 	ZEN_TRACE_CPU("httpsys::HandleCompletion");
 
 	// Route request
@@ -2401,6 +2410,18 @@ HttpSysServer::OnGetExternalHost() const
 	}
 }
 
+uint64_t
+HttpSysServer::GetTotalBytesReceived() const
+{
+	return m_TotalBytesReceived.load(std::memory_order_relaxed);
+}
+
+uint64_t
+HttpSysServer::GetTotalBytesSent() const
+{
+	return m_TotalBytesSent.load(std::memory_order_relaxed);
+}
+
 void
 HttpSysServer::OnRegisterService(HttpService& Service)
 {