From d0a07e555577dcd4a8f55f1b45d9e8e4e6366ab7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 10 Mar 2026 17:27:26 +0100 Subject: HttpClient using libcurl, Unix Sockets for HTTP. HTTPS support (#770) The main goal of this change is to eliminate the cpr back-end altogether and replace it with the curl implementation. I would expect to drop cpr as soon as we feel happy with the libcurl back-end. That would leave us with a direct dependency on libcurl only, and cpr can be eliminated as a dependency. ### HttpClient Backend Overhaul - Implemented a new **libcurl-based HttpClient** backend (`httpclientcurl.cpp`, ~2000 lines) as an alternative to the cpr-based one - Made HttpClient backend **configurable at runtime** via constructor arguments and `-httpclient=...` CLI option (for zen, zenserver, and tests) - Extended HttpClient test suite to cover multipart/content-range scenarios ### Unix Domain Socket Support - Added Unix domain socket support to **httpasio** (server side) - Added Unix domain socket support to **HttpClient** - Added Unix domain socket support to **HttpWsClient** (WebSocket client) - Templatized `HttpServerConnectionT` and `WsAsioConnectionT` to handle TCP, Unix, and SSL sockets uniformly via `if constexpr` dispatch ### HTTPS Support - Added **preliminary HTTPS support to httpasio** (for Mac/Linux via OpenSSL) - Added **basic HTTPS support for http.sys** (Windows) - Implemented HTTPS test for httpasio - Split `InitializeServer` into smaller sub-functions for http.sys ### Other Notable Changes - Improved **zenhttp-test stability** with dynamic port allocation - Enhanced port retry logic in http.sys (handles ERROR_ACCESS_DENIED) - Fatal signal/exception handlers for backtrace generation in tests - Added `zen bench http` subcommand to exercise network + HTTP client/server communication stack --- src/zenhttp/servers/httpsys.cpp | 409 ++++++++++++++++++++++++++++++++++------ 1 file changed, 353 insertions(+), 56 deletions(-) (limited to 'src/zenhttp/servers/httpsys.cpp') diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index dfe6bb6aa..83b98013e 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -116,6 +116,12 @@ public: private: int InitializeServer(int BasePort); + bool CreateSessionAndUrlGroup(); + bool RegisterLocalUrls(std::u8string_view Scheme, int Port, std::vector& OutUris); + int RegisterHttpUrls(int BasePort); + bool RegisterHttpsUrls(); + bool CreateRequestQueue(int EffectivePort); + bool SetupIoCompletionPort(); void Cleanup(); void StartServer(); @@ -125,6 +131,9 @@ private: void RegisterService(const char* Endpoint, HttpService& Service); void UnregisterService(const char* Endpoint, HttpService& Service); + bool BindSslCertificate(int Port); + void UnbindSslCertificate(); + private: LoggerRef m_Log; LoggerRef m_RequestLog; @@ -140,7 +149,10 @@ private: RwLock m_AsyncWorkPoolInitLock; std::atomic m_AsyncWorkPool = nullptr; - std::vector m_BaseUris; // eg: http://*:nnnn/ + std::vector m_BaseUris; // eg: http://*:nnnn/ + std::vector m_HttpsBaseUris; // eg: https://*:nnnn/ + bool m_DidAutoBindCert = false; + int m_HttpsPort = 0; HTTP_SERVER_SESSION_ID m_HttpSessionId = 0; HTTP_URL_GROUP_ID m_HttpUrlGroupId = 0; HANDLE m_RequestQueueHandle = 0; @@ -1082,39 +1094,63 @@ HttpSysServer::OnClose() } } -int -HttpSysServer::InitializeServer(int BasePort) +bool +HttpSysServer::CreateSessionAndUrlGroup() { - ZEN_MEMSCOPE(GetHttpsysTag()); - - using namespace std::literals; - - WideStringBuilder<64> WildcardUrlPath; - WildcardUrlPath << u8"http://*:"sv << int64_t(BasePort) << u8"/"sv; - - m_IsOk = false; - ULONG Result = HttpCreateServerSession(HTTPAPI_VERSION_2, &m_HttpSessionId, 0); if (Result != NO_ERROR) { - ZEN_ERROR("Failed to create server session for '{}': {} ({:#x})", - WideToUtf8(WildcardUrlPath), - GetSystemErrorAsString(Result), - Result); + ZEN_ERROR("Failed to create server session: {} ({:#x})", GetSystemErrorAsString(Result), Result); - return 0; + return false; } Result = HttpCreateUrlGroup(m_HttpSessionId, &m_HttpUrlGroupId, 0); if (Result != NO_ERROR) { - ZEN_ERROR("Failed to create URL group for '{}': {} ({:#x})", WideToUtf8(WildcardUrlPath), GetSystemErrorAsString(Result), Result); + ZEN_ERROR("Failed to create URL group: {} ({:#x})", GetSystemErrorAsString(Result), Result); - return 0; + return false; } + return true; +} + +bool +HttpSysServer::RegisterLocalUrls(std::u8string_view Scheme, int Port, std::vector& OutUris) +{ + using namespace std::literals; + + const std::u8string_view Hosts[] = {u8"[::1]"sv, u8"localhost"sv, u8"127.0.0.1"sv}; + + for (const std::u8string_view Host : Hosts) + { + WideStringBuilder<64> LocalUrl; + LocalUrl << Scheme << u8"://"sv << Host << u8":"sv << int64_t(Port) << u8"/"sv; + + ULONG Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, LocalUrl.c_str(), HTTP_URL_CONTEXT(0), 0); + + if (Result == NO_ERROR) + { + ZEN_WARN("Registered local-only handler '{}' - this is not accessible from remote hosts", WideToUtf8(LocalUrl)); + OutUris.push_back(LocalUrl.c_str()); + } + else + { + break; + } + } + + return !OutUris.empty(); +} + +int +HttpSysServer::RegisterHttpUrls(int BasePort) +{ + using namespace std::literals; + m_BaseUris.clear(); const bool AllowPortProbing = !m_InitialConfig.IsDedicatedServer; @@ -1122,6 +1158,11 @@ HttpSysServer::InitializeServer(int BasePort) int EffectivePort = BasePort; + WideStringBuilder<64> WildcardUrlPath; + WildcardUrlPath << u8"http://*:"sv << int64_t(BasePort) << u8"/"sv; + + ULONG Result; + if (m_InitialConfig.ForceLoopback) { // Force trigger of opening using local port @@ -1177,11 +1218,11 @@ HttpSysServer::InitializeServer(int BasePort) { if (AllowLocalOnly) { - // If we can't register the wildcard path, we fall back to local paths - // This local paths allow requests originating locally to function, but will not allow - // remote origin requests to function. This can be remedied by using netsh + // If we can't register the wildcard path, we fall back to local paths. + // Local paths allow requests originating locally to function, but will not allow + // remote origin requests to function. This can be remedied by using netsh // during an install process to grant permissions to route public access to the appropriate - // port for the current user. eg: + // port for the current user. eg: // netsh http add urlacl url=http://*:8558/ user= if (!m_InitialConfig.ForceLoopback) @@ -1246,7 +1287,7 @@ HttpSysServer::InitializeServer(int BasePort) } } - if (m_BaseUris.empty()) + if (m_BaseUris.empty() && m_InitialConfig.HttpsPort == 0) { ZEN_ERROR("Failed to add base URL to URL group for '{}': {} ({:#x})", WideToUtf8(WildcardUrlPath), @@ -1256,16 +1297,104 @@ HttpSysServer::InitializeServer(int BasePort) return 0; } + return EffectivePort; +} + +bool +HttpSysServer::RegisterHttpsUrls() +{ + using namespace std::literals; + + const bool AllowLocalOnly = !m_InitialConfig.IsDedicatedServer; + const int HttpsPort = m_InitialConfig.HttpsPort; + + // If HTTPS-only mode, remove HTTP URLs and clear base URIs + if (m_InitialConfig.HttpsOnly) + { + for (const std::wstring& Uri : m_BaseUris) + { + HttpRemoveUrlFromUrlGroup(m_HttpUrlGroupId, Uri.c_str(), 0); + } + m_BaseUris.clear(); + } + + // Auto-bind certificate if thumbprint is provided + if (!m_InitialConfig.CertThumbprint.empty()) + { + if (!BindSslCertificate(HttpsPort)) + { + return false; + } + } + else + { + ZEN_INFO("HTTPS port {} configured without thumbprint - assuming pre-registered SSL certificate", HttpsPort); + } + + // Register HTTPS URLs using same pattern as HTTP + + WideStringBuilder<64> HttpsWildcard; + HttpsWildcard << u8"https://*:"sv << int64_t(HttpsPort) << u8"/"sv; + + ULONG HttpsResult = NO_ERROR; + + if (m_InitialConfig.ForceLoopback) + { + HttpsResult = ERROR_ACCESS_DENIED; + } + else + { + HttpsResult = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, HttpsWildcard.c_str(), HTTP_URL_CONTEXT(0), 0); + } + + if (HttpsResult == NO_ERROR) + { + m_HttpsBaseUris.push_back(HttpsWildcard.c_str()); + } + else if (HttpsResult == ERROR_ACCESS_DENIED && AllowLocalOnly) + { + if (!m_InitialConfig.ForceLoopback) + { + ZEN_WARN( + "Unable to register HTTPS handler using '{}' - falling back to local-only. " + "Please ensure the appropriate netsh URL reservation and SSL certificate configuration is made.", + WideToUtf8(HttpsWildcard)); + } + + RegisterLocalUrls(u8"https", HttpsPort, m_HttpsBaseUris); + } + else if (HttpsResult != NO_ERROR) + { + ZEN_ERROR("Failed to register HTTPS URL '{}': {} ({:#x})", + WideToUtf8(HttpsWildcard), + GetSystemErrorAsString(HttpsResult), + HttpsResult); + return false; + } + + if (m_HttpsBaseUris.empty()) + { + ZEN_ERROR("Failed to register any HTTPS URL for port {}", HttpsPort); + return false; + } + + m_HttpsPort = HttpsPort; + return true; +} + +bool +HttpSysServer::CreateRequestQueue(int EffectivePort) +{ HTTP_BINDING_INFO HttpBindingInfo = {{0}, 0}; WideStringBuilder<64> QueueName; QueueName << "zenserver_" << EffectivePort; - Result = HttpCreateRequestQueue(HTTPAPI_VERSION_2, - /* Name */ QueueName.c_str(), - /* SecurityAttributes */ nullptr, - /* Flags */ 0, - &m_RequestQueueHandle); + ULONG Result = HttpCreateRequestQueue(HTTPAPI_VERSION_2, + /* Name */ QueueName.c_str(), + /* SecurityAttributes */ nullptr, + /* Flags */ 0, + &m_RequestQueueHandle); if (Result != NO_ERROR) { @@ -1274,7 +1403,7 @@ HttpSysServer::InitializeServer(int BasePort) GetSystemErrorAsString(Result), Result); - return 0; + return false; } HttpBindingInfo.Flags.Present = 1; @@ -1289,7 +1418,7 @@ HttpSysServer::InitializeServer(int BasePort) GetSystemErrorAsString(Result), Result); - return 0; + return false; } // Configure rejection method. Default is to drop the connection, it's better if we @@ -1323,22 +1452,77 @@ HttpSysServer::InitializeServer(int BasePort) } } - // Create I/O completion port + return true; +} +bool +HttpSysServer::SetupIoCompletionPort() +{ std::error_code ErrorCode; m_IoThreadPool->CreateIocp(m_RequestQueueHandle, HttpSysTransaction::IoCompletionCallback, /* Context */ this, /* out */ ErrorCode); if (ErrorCode) { - ZEN_ERROR("Failed to create IOCP for '{}': {}", WideToUtf8(m_BaseUris.front()), ErrorCode.message()); + ZEN_ERROR("Failed to create IOCP: {}", ErrorCode.message()); + return false; + } + m_IsOk = true; + + if (!m_BaseUris.empty()) + { + ZEN_INFO("Started http.sys server at '{}'", WideToUtf8(m_BaseUris.front())); + } + if (!m_HttpsBaseUris.empty()) + { + ZEN_INFO("Started http.sys HTTPS server at '{}'", WideToUtf8(m_HttpsBaseUris.front())); + } + + return true; +} + +int +HttpSysServer::InitializeServer(int BasePort) +{ + ZEN_MEMSCOPE(GetHttpsysTag()); + + m_IsOk = false; + + if (!CreateSessionAndUrlGroup()) + { return 0; } - else + + int EffectivePort = RegisterHttpUrls(BasePort); + + if (m_InitialConfig.HttpsPort > 0) + { + if (!RegisterHttpsUrls()) + { + return 0; + } + } + + if (m_BaseUris.empty() && m_HttpsBaseUris.empty()) { - m_IsOk = true; + ZEN_ERROR("No HTTP or HTTPS listeners could be registered"); + return 0; + } - ZEN_INFO("Started http.sys server at '{}'", WideToUtf8(m_BaseUris.front())); + if (!CreateRequestQueue(EffectivePort)) + { + return 0; + } + + if (!SetupIoCompletionPort()) + { + return 0; + } + + // When HTTPS-only, return the HTTPS port as the effective port + if (m_InitialConfig.HttpsOnly && m_HttpsPort > 0) + { + return m_HttpsPort; } return EffectivePort; @@ -1349,6 +1533,8 @@ HttpSysServer::Cleanup() { ++m_IsShuttingDown; + UnbindSslCertificate(); + if (m_RequestQueueHandle) { HttpCloseRequestQueue(m_RequestQueueHandle); @@ -1368,6 +1554,105 @@ HttpSysServer::Cleanup() } } +// {7E3F4B2A-1C8D-4A6E-B5F0-9D2E8C7A3B1F} - Fixed GUID for zenserver SSL bindings +static constexpr GUID ZenServerSslAppId = {0x7E3F4B2A, 0x1C8D, 0x4A6E, {0xB5, 0xF0, 0x9D, 0x2E, 0x8C, 0x7A, 0x3B, 0x1F}}; + +bool +HttpSysServer::BindSslCertificate(int Port) +{ + const std::string& Thumbprint = m_InitialConfig.CertThumbprint; + if (Thumbprint.size() != 40) + { + ZEN_ERROR("SSL certificate thumbprint must be exactly 40 hex characters, got {}", Thumbprint.size()); + return false; + } + + BYTE CertHash[20] = {}; + if (!ParseHexBytes(Thumbprint, CertHash)) + { + ZEN_ERROR("SSL certificate thumbprint contains invalid hex characters"); + return false; + } + + SOCKADDR_IN Address = {}; + Address.sin_family = AF_INET; + Address.sin_port = htons(static_cast(Port)); + Address.sin_addr.s_addr = INADDR_ANY; + + const std::wstring StoreNameW = UTF8_to_UTF16(m_InitialConfig.CertStoreName.c_str()); + + HTTP_SERVICE_CONFIG_SSL_SET SslConfig = {}; + SslConfig.KeyDesc.pIpPort = reinterpret_cast(&Address); + SslConfig.ParamDesc.pSslHash = CertHash; + SslConfig.ParamDesc.SslHashLength = sizeof(CertHash); + SslConfig.ParamDesc.pSslCertStoreName = const_cast(StoreNameW.c_str()); + SslConfig.ParamDesc.AppId = ZenServerSslAppId; + + ULONG Result = HttpSetServiceConfiguration(0, HttpServiceConfigSSLCertInfo, &SslConfig, sizeof(SslConfig), nullptr); + + if (Result == ERROR_ALREADY_EXISTS) + { + // Remove existing binding and retry + HTTP_SERVICE_CONFIG_SSL_SET DeleteConfig = {}; + DeleteConfig.KeyDesc.pIpPort = reinterpret_cast(&Address); + + HttpDeleteServiceConfiguration(0, HttpServiceConfigSSLCertInfo, &DeleteConfig, sizeof(DeleteConfig), nullptr); + + Result = HttpSetServiceConfiguration(0, HttpServiceConfigSSLCertInfo, &SslConfig, sizeof(SslConfig), nullptr); + } + + if (Result != NO_ERROR) + { + ZEN_ERROR( + "Failed to bind SSL certificate to port {}: {} ({:#x}). " + "This operation may require running as administrator.", + Port, + GetSystemErrorAsString(Result), + Result); + return false; + } + + m_DidAutoBindCert = true; + m_HttpsPort = Port; + + ZEN_INFO("SSL certificate auto-bound for 0.0.0.0:{} (thumbprint: {}..., store: {})", + Port, + Thumbprint.substr(0, 8), + m_InitialConfig.CertStoreName); + + return true; +} + +void +HttpSysServer::UnbindSslCertificate() +{ + if (!m_DidAutoBindCert) + { + return; + } + + SOCKADDR_IN Address = {}; + Address.sin_family = AF_INET; + Address.sin_port = htons(static_cast(m_HttpsPort)); + Address.sin_addr.s_addr = INADDR_ANY; + + HTTP_SERVICE_CONFIG_SSL_SET SslConfig = {}; + SslConfig.KeyDesc.pIpPort = reinterpret_cast(&Address); + + ULONG Result = HttpDeleteServiceConfiguration(0, HttpServiceConfigSSLCertInfo, &SslConfig, sizeof(SslConfig), nullptr); + + if (Result != NO_ERROR) + { + ZEN_WARN("Failed to remove SSL certificate binding from port {}: {} ({:#x})", m_HttpsPort, GetSystemErrorAsString(Result), Result); + } + else + { + ZEN_INFO("SSL certificate binding removed from port {}", m_HttpsPort); + } + + m_DidAutoBindCert = false; +} + WorkerThreadPool& HttpSysServer::WorkPool() { @@ -1495,19 +1780,23 @@ HttpSysServer::RegisterService(const char* UrlPath, HttpService& Service) // Convert to wide string - for (const std::wstring& BaseUri : m_BaseUris) - { - std::wstring Url16 = BaseUri + PathUtf16; - - ULONG Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, Url16.c_str(), HTTP_URL_CONTEXT(&Service), 0 /* Reserved */); - - if (Result != NO_ERROR) + auto RegisterWithBaseUris = [&](const std::vector& BaseUris) { + for (const std::wstring& BaseUri : BaseUris) { - ZEN_ERROR("HttpAddUrlToUrlGroup failed with result: '{}'", GetSystemErrorAsString(Result)); + std::wstring Url16 = BaseUri + PathUtf16; - return; + ULONG Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, Url16.c_str(), HTTP_URL_CONTEXT(&Service), 0 /* Reserved */); + + if (Result != NO_ERROR) + { + ZEN_ERROR("HttpAddUrlToUrlGroup failed with result: '{}'", GetSystemErrorAsString(Result)); + return; + } } - } + }; + + RegisterWithBaseUris(m_BaseUris); + RegisterWithBaseUris(m_HttpsBaseUris); } void @@ -1522,19 +1811,22 @@ HttpSysServer::UnregisterService(const char* UrlPath, HttpService& Service) const std::wstring PathUtf16 = UTF8_to_UTF16(UrlPath); - // Convert to wide string - - for (const std::wstring& BaseUri : m_BaseUris) - { - std::wstring Url16 = BaseUri + PathUtf16; + auto UnregisterFromBaseUris = [&](const std::vector& BaseUris) { + for (const std::wstring& BaseUri : BaseUris) + { + std::wstring Url16 = BaseUri + PathUtf16; - ULONG Result = HttpRemoveUrlFromUrlGroup(m_HttpUrlGroupId, Url16.c_str(), 0); + ULONG Result = HttpRemoveUrlFromUrlGroup(m_HttpUrlGroupId, Url16.c_str(), 0); - if (Result != NO_ERROR) - { - ZEN_ERROR("HttpRemoveUrlFromUrlGroup failed with result: '{}'", GetSystemErrorAsString(Result)); + if (Result != NO_ERROR) + { + ZEN_ERROR("HttpRemoveUrlFromUrlGroup failed with result: '{}'", GetSystemErrorAsString(Result)); + } } - } + }; + + UnregisterFromBaseUris(m_BaseUris); + UnregisterFromBaseUris(m_HttpsBaseUris); } ////////////////////////////////////////////////////////////////////////// @@ -2422,6 +2714,11 @@ HttpSysServer::OnInitialize(int BasePort, std::filesystem::path DataDir) ZEN_UNUSED(DataDir); if (int EffectivePort = InitializeServer(BasePort)) { + if (m_HttpsPort > 0) + { + SetEffectiveHttpsPort(m_HttpsPort); + } + StartServer(); return EffectivePort; -- cgit v1.2.3