From d0a07e555577dcd4a8f55f1b45d9e8e4e6366ab7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 10 Mar 2026 17:27:26 +0100 Subject: HttpClient using libcurl, Unix Sockets for HTTP. HTTPS support (#770) The main goal of this change is to eliminate the cpr back-end altogether and replace it with the curl implementation. I would expect to drop cpr as soon as we feel happy with the libcurl back-end. That would leave us with a direct dependency on libcurl only, and cpr can be eliminated as a dependency. ### HttpClient Backend Overhaul - Implemented a new **libcurl-based HttpClient** backend (`httpclientcurl.cpp`, ~2000 lines) as an alternative to the cpr-based one - Made HttpClient backend **configurable at runtime** via constructor arguments and `-httpclient=...` CLI option (for zen, zenserver, and tests) - Extended HttpClient test suite to cover multipart/content-range scenarios ### Unix Domain Socket Support - Added Unix domain socket support to **httpasio** (server side) - Added Unix domain socket support to **HttpClient** - Added Unix domain socket support to **HttpWsClient** (WebSocket client) - Templatized `HttpServerConnectionT` and `WsAsioConnectionT` to handle TCP, Unix, and SSL sockets uniformly via `if constexpr` dispatch ### HTTPS Support - Added **preliminary HTTPS support to httpasio** (for Mac/Linux via OpenSSL) - Added **basic HTTPS support for http.sys** (Windows) - Implemented HTTPS test for httpasio - Split `InitializeServer` into smaller sub-functions for http.sys ### Other Notable Changes - Improved **zenhttp-test stability** with dynamic port allocation - Enhanced port retry logic in http.sys (handles ERROR_ACCESS_DENIED) - Fatal signal/exception handlers for backtrace generation in tests - Added `zen bench http` subcommand to exercise network + HTTP client/server communication stack --- src/zenserver/compute/computeserver.cpp | 2 +- src/zenserver/config/config.cpp | 122 +++++++++++++++++++++ src/zenserver/config/config.h | 6 + src/zenserver/sessions/httpsessions.cpp | 4 + src/zenserver/sessions/sessions.cpp | 4 +- .../storage/cache/httpstructuredcache.cpp | 4 - src/zenserver/zenserver.cpp | 9 ++ 7 files changed, 144 insertions(+), 7 deletions(-) (limited to 'src/zenserver') diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp index c64f081b3..2ac3de599 100644 --- a/src/zenserver/compute/computeserver.cpp +++ b/src/zenserver/compute/computeserver.cpp @@ -674,7 +674,7 @@ ZenComputeServer::PostAnnounce() { ZEN_ERROR("failed to notify coordinator at '{}': HTTP error {} - {}", m_CoordinatorEndpoint, - Result.Error->ErrorCode, + static_cast(Result.Error->ErrorCode), Result.Error->ErrorMessage); } else if (!IsHttpOk(Result.StatusCode)) diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp index e36352dae..ef9c6b7b8 100644 --- a/src/zenserver/config/config.cpp +++ b/src/zenserver/config/config.cpp @@ -144,10 +144,15 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions ////// network + LuaOptions.AddOption("network.httpclientbackend"sv, ServerOptions.HttpClient.Backend, "httpclient"sv); LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpConfig.ServerClass, "http"sv); LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpConfig.ThreadCount, "http-threads"sv); LuaOptions.AddOption("network.port"sv, ServerOptions.BasePort, "port"sv); LuaOptions.AddOption("network.forceloopback"sv, ServerOptions.HttpConfig.ForceLoopback, "http-forceloopback"sv); + LuaOptions.AddOption("network.unixsocket"sv, ServerOptions.HttpConfig.UnixSocketPath, "unix-socket"sv); + LuaOptions.AddOption("network.https.port"sv, ServerOptions.HttpConfig.HttpsPort, "https-port"sv); + LuaOptions.AddOption("network.https.certfile"sv, ServerOptions.HttpConfig.CertFile, "cert-file"sv); + LuaOptions.AddOption("network.https.keyfile"sv, ServerOptions.HttpConfig.KeyFile, "key-file"sv); #if ZEN_WITH_HTTPSYS LuaOptions.AddOption("network.httpsys.async.workthreads"sv, @@ -159,6 +164,10 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions LuaOptions.AddOption("network.httpsys.requestlogging"sv, ServerOptions.HttpConfig.HttpSys.IsRequestLoggingEnabled, "httpsys-enable-request-logging"sv); + LuaOptions.AddOption("network.httpsys.httpsport"sv, ServerOptions.HttpConfig.HttpSys.HttpsPort, "httpsys-https-port"sv); + LuaOptions.AddOption("network.httpsys.certthumbprint"sv, ServerOptions.HttpConfig.HttpSys.CertThumbprint, "httpsys-cert-thumbprint"sv); + LuaOptions.AddOption("network.httpsys.certstorename"sv, ServerOptions.HttpConfig.HttpSys.CertStoreName, "httpsys-cert-store"sv); + LuaOptions.AddOption("network.httpsys.httpsonly"sv, ServerOptions.HttpConfig.HttpSys.HttpsOnly, "httpsys-https-only"sv); #endif #if ZEN_WITH_TRACE @@ -302,6 +311,34 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi cxxopts::value(ServerOptions.HttpConfig.ForceLoopback)->default_value("false"), ""); + options.add_option("network", + "", + "unix-socket", + "Unix domain socket path to listen on (in addition to TCP)", + cxxopts::value(ServerOptions.HttpConfig.UnixSocketPath), + ""); + + options.add_option("network", + "", + "https-port", + "HTTPS listen port (0 = disabled)", + cxxopts::value(ServerOptions.HttpConfig.HttpsPort)->default_value("0"), + ""); + + options.add_option("network", + "", + "cert-file", + "Path to PEM certificate chain file for HTTPS", + cxxopts::value(ServerOptions.HttpConfig.CertFile), + ""); + + options.add_option("network", + "", + "key-file", + "Path to PEM private key file for HTTPS", + cxxopts::value(ServerOptions.HttpConfig.KeyFile), + ""); + options.add_option("network", "", "security-config-path", @@ -330,8 +367,43 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi "Enables Httpsys request logging", cxxopts::value(ServerOptions.HttpConfig.HttpSys.IsRequestLoggingEnabled), ""); + + options.add_option("httpsys", + "", + "httpsys-https-port", + "HTTPS listen port for http.sys (0 = disabled)", + cxxopts::value(ServerOptions.HttpConfig.HttpSys.HttpsPort)->default_value("0"), + ""); + + options.add_option("httpsys", + "", + "httpsys-cert-thumbprint", + "SHA-1 certificate thumbprint for auto SSL binding", + cxxopts::value(ServerOptions.HttpConfig.HttpSys.CertThumbprint), + ""); + + options.add_option("httpsys", + "", + "httpsys-cert-store", + "Windows certificate store name for SSL binding", + cxxopts::value(ServerOptions.HttpConfig.HttpSys.CertStoreName)->default_value("MY"), + ""); + + options.add_option("httpsys", + "", + "httpsys-https-only", + "Disable HTTP listener when HTTPS is active", + cxxopts::value(ServerOptions.HttpConfig.HttpSys.HttpsOnly)->default_value("false"), + ""); #endif + options.add_option("network", + "", + "httpclient", + "Select HTTP client implementation (e.g. 'curl', 'cpr')", + cxxopts::value(ServerOptions.HttpClient.Backend)->default_value("cpr"), + ""); + options.add_option("network", "", "http", @@ -397,6 +469,56 @@ ZenServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenServerConfig ServerOptions.SecurityConfigPath = MakeSafeAbsolutePath(SecurityConfigPath); LoggingOptions.ApplyOptions(ServerOptions.LoggingConfig); + +#if ZEN_WITH_HTTPSYS + // Validate HTTPS options + const auto& HttpSys = ServerOptions.HttpConfig.HttpSys; + if (HttpSys.HttpsOnly && HttpSys.HttpsPort == 0) + { + throw OptionParseException("'--httpsys-https-only' requires '--httpsys-https-port' to be set", options.help()); + } + if (!HttpSys.CertThumbprint.empty() && HttpSys.CertThumbprint.size() != 40) + { + throw OptionParseException("'--httpsys-cert-thumbprint' must be exactly 40 hex characters (SHA-1)", options.help()); + } + if (!HttpSys.CertThumbprint.empty()) + { + for (char Ch : HttpSys.CertThumbprint) + { + if (!((Ch >= '0' && Ch <= '9') || (Ch >= 'a' && Ch <= 'f') || (Ch >= 'A' && Ch <= 'F'))) + { + throw OptionParseException("'--httpsys-cert-thumbprint' contains non-hex characters", options.help()); + } + } + } + if (HttpSys.HttpsPort > 0 && HttpSys.HttpsPort == ServerOptions.BasePort && !HttpSys.HttpsOnly) + { + throw OptionParseException("'--httpsys-https-port' must differ from '--port' when both HTTP and HTTPS are active", options.help()); + } +#endif + + // Validate generic HTTPS options (used by ASIO backend) + if (ServerOptions.HttpConfig.HttpsPort > 0) + { + if (ServerOptions.HttpConfig.CertFile.empty() || ServerOptions.HttpConfig.KeyFile.empty()) + { + throw OptionParseException("'--https-port' requires both '--cert-file' and '--key-file' to be set", options.help()); + } + if (!std::filesystem::exists(ServerOptions.HttpConfig.CertFile)) + { + throw OptionParseException(fmt::format("'--cert-file' path '{}' does not exist", ServerOptions.HttpConfig.CertFile), + options.help()); + } + if (!std::filesystem::exists(ServerOptions.HttpConfig.KeyFile)) + { + throw OptionParseException(fmt::format("'--key-file' path '{}' does not exist", ServerOptions.HttpConfig.KeyFile), + options.help()); + } + if (ServerOptions.HttpConfig.HttpsPort == ServerOptions.BasePort) + { + throw OptionParseException("'--https-port' must differ from '--port'", options.help()); + } + } } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h index 55aee07f9..88226f810 100644 --- a/src/zenserver/config/config.h +++ b/src/zenserver/config/config.h @@ -38,8 +38,14 @@ struct ZenSentryConfig bool Debug = false; // Enable debug mode for Sentry }; +struct HttpClientConfig +{ + std::string Backend = "cpr"; // Choice of HTTP client implementation (e.g. "curl", "cpr") +}; + struct ZenServerConfig { + HttpClientConfig HttpClient; HttpServerConfig HttpConfig; ZenSentryConfig SentryConfig; ZenStatsConfig StatsConfig; diff --git a/src/zenserver/sessions/httpsessions.cpp b/src/zenserver/sessions/httpsessions.cpp index 05be3c814..6cf12bea4 100644 --- a/src/zenserver/sessions/httpsessions.cpp +++ b/src/zenserver/sessions/httpsessions.cpp @@ -258,6 +258,10 @@ HttpSessionsService::SessionRequest(HttpRouterRequest& Req) } return ServerRequest.WriteResponse(HttpResponseCode::NotFound); } + default: + { + return ServerRequest.WriteResponse(HttpResponseCode::MethodNotAllowed); + } } } diff --git a/src/zenserver/sessions/sessions.cpp b/src/zenserver/sessions/sessions.cpp index f73aa40ff..d919db6e9 100644 --- a/src/zenserver/sessions/sessions.cpp +++ b/src/zenserver/sessions/sessions.cpp @@ -64,6 +64,8 @@ SessionsService::RegisterSession(const Oid& SessionId, std::string AppName, cons return false; } + ZEN_INFO("Session {} registered (AppName: {}, JobId: {})", SessionId, AppName, JobId); + const DateTime Now = DateTime::Now(); m_Sessions.emplace(SessionId, Ref(new Session(SessionInfo{.Id = SessionId, @@ -72,8 +74,6 @@ SessionsService::RegisterSession(const Oid& SessionId, std::string AppName, cons .Metadata = CbObject::Clone(Metadata), .CreatedAt = Now, .UpdatedAt = Now}))); - - ZEN_INFO("Session {} registered (AppName: {}, JobId: {})", SessionId, AppName, JobId); return true; } diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp index 06b8f6c27..bbdb03ba4 100644 --- a/src/zenserver/storage/cache/httpstructuredcache.cpp +++ b/src/zenserver/storage/cache/httpstructuredcache.cpp @@ -1892,8 +1892,6 @@ HttpStructuredCacheService::CollectStats() { Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount; - Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); - Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); } Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; @@ -2025,8 +2023,6 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) { Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount; - Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); - Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); } Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 88b85d7d9..49ae1b6ff 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -146,6 +147,14 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState:: EnqueueSigIntTimer(); + // Configure HTTP client back-end + + const std::string HttpClientBackend = ToLower(ServerOptions.HttpClient.Backend); + zen::SetDefaultHttpClientBackend(HttpClientBackend); + ZEN_INFO("Using '{}' as HTTP client backend", HttpClientBackend); + + // Initialize HTTP server + m_Http = CreateHttpServer(ServerOptions.HttpConfig); int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort, ServerOptions.DataDir); if (EffectiveBasePort == 0) -- cgit v1.2.3