diff options
| author | Stefan Boberg <[email protected]> | 2026-03-23 12:54:14 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2026-03-23 12:54:14 +0100 |
| commit | 8e2c307bdb501db0ab0ce2d51bc61b552855ee11 (patch) | |
| tree | 8f9be7e926bc555318a68794ee75ad5ad0dd979f /src/zenhttp | |
| parent | Logger simplification (#883) (diff) | |
| download | zen-8e2c307bdb501db0ab0ce2d51bc61b552855ee11.tar.xz zen-8e2c307bdb501db0ab0ce2d51bc61b552855ee11.zip | |
Unique session/client tracking using HyperLogLog (#884)
## Summary
Adds probabilistic cardinality estimation for tracking unique HTTP clients and sessions using a HyperLogLog implementation.
- Add a `HyperLogLog<Precision>` template in `zentelemetry` with thread-safe lock-free register updates, merge support, and XXH3 hashing
- Feed client IP addresses (via raw bytes) and session IDs (via `Oid` bytes) into their respective HyperLogLog estimators from both the ASIO and http.sys server backends
- Emit `distinct_clients` and `distinct_sessions` cardinality estimates in HTTP `CollectStats()`
- Add tests covering empty, single, duplicates, accuracy, merge, and clear scenarios
## Why HyperLogLog
Tracking exact unique counts would require storing every observed IP or session ID. HyperLogLog provides a memory-bounded probabilistic estimate (~1–2% error) using only a few KB of memory regardless of traffic volume.
Diffstat (limited to 'src/zenhttp')
| -rw-r--r-- | src/zenhttp/httpserver.cpp | 3 | ||||
| -rw-r--r-- | src/zenhttp/include/zenhttp/httpserver.h | 16 | ||||
| -rw-r--r-- | src/zenhttp/servers/httpasio.cpp | 28 | ||||
| -rw-r--r-- | src/zenhttp/servers/httpsys.cpp | 17 |
4 files changed, 62 insertions, 2 deletions
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index e5cfbcbae..a46c5b851 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -988,6 +988,9 @@ HttpServer::CollectStats() } Cbo.EndObject(); + Cbo << "distinct_clients" << m_ClientAddresses.Count(); + Cbo << "distinct_sessions" << m_ClientSessions.Count(); + Cbo.BeginObject("websockets"); { Cbo << "active_connections" << GetActiveWebSocketConnectionCount(); diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index a7d7f4d9c..633eb06be 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -13,6 +13,7 @@ #include <zencore/uid.h> #include <zenhttp/httpcommon.h> +#include <zentelemetry/hyperloglog.h> #include <zentelemetry/stats.h> #include <filesystem> @@ -265,6 +266,19 @@ public: /** Mark that a request has been handled. Called by server implementations. */ void MarkRequest() { m_RequestMeter.Mark(); } + /** Record a client address for distinct-client tracking. Pass the raw + * address bytes (4 bytes for IPv4, 16 for IPv6) to avoid string conversion. */ + void MarkClientAddress(const void* AddressBytes, size_t Size) { m_ClientAddresses.Add(AddressBytes, Size); } + + /** Record a session ID for distinct-session tracking. */ + void MarkSessionId(const Oid& SessionId) + { + if (SessionId) + { + m_ClientSessions.Add(&SessionId.OidBits, sizeof(SessionId.OidBits)); + } + } + /** Set a default redirect path for root requests */ void SetDefaultRedirect(std::string_view Path) { m_DefaultRedirect = Path; } @@ -297,6 +311,8 @@ private: int m_EffectiveHttpsPort = 0; std::string m_ExternalHost; metrics::Meter m_RequestMeter; + metrics::HyperLogLog<12> m_ClientAddresses; // ~4 KiB, ~1.6% error — sufficient for client counting + metrics::HyperLogLog<12> m_ClientSessions; std::string m_DefaultRedirect; std::atomic<uint64_t> m_ActiveWebSocketConnections{0}; std::atomic<uint64_t> m_WsFramesReceived{0}; diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index a2cae8762..7972777b8 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -1330,14 +1330,36 @@ HttpServerConnectionT<SocketType>::HandleRequest() { auto RemoteEndpoint = m_Socket->remote_endpoint(); IsLocalConnection = m_Socket->local_endpoint().address() == RemoteEndpoint.address(); - RemoteAddress = RemoteEndpoint.address().to_string(); + auto Addr = RemoteEndpoint.address(); + RemoteAddress = Addr.to_string(); + if (Addr.is_v4()) + { + auto Bytes = Addr.to_v4().to_bytes(); + m_Server.m_HttpServer->MarkClientAddress(Bytes.data(), Bytes.size()); + } + else + { + auto Bytes = Addr.to_v6().to_bytes(); + m_Server.m_HttpServer->MarkClientAddress(Bytes.data(), Bytes.size()); + } } #if ZEN_USE_OPENSSL else if constexpr (std::is_same_v<SocketType, SslSocket>) { auto RemoteEndpoint = m_Socket->lowest_layer().remote_endpoint(); IsLocalConnection = m_Socket->lowest_layer().local_endpoint().address() == RemoteEndpoint.address(); - RemoteAddress = RemoteEndpoint.address().to_string(); + auto Addr = RemoteEndpoint.address(); + RemoteAddress = Addr.to_string(); + if (Addr.is_v4()) + { + auto Bytes = Addr.to_v4().to_bytes(); + m_Server.m_HttpServer->MarkClientAddress(Bytes.data(), Bytes.size()); + } + else + { + auto Bytes = Addr.to_v6().to_bytes(); + m_Server.m_HttpServer->MarkClientAddress(Bytes.data(), Bytes.size()); + } } #endif else @@ -1345,6 +1367,8 @@ HttpServerConnectionT<SocketType>::HandleRequest() RemoteAddress = "unix"; } + m_Server.m_HttpServer->MarkSessionId(m_RequestData.SessionId()); + HttpAsioServerRequest Request(m_RequestData, *Service, m_RequestData.Body(), diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 9fe9a2254..2cad97725 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -2020,6 +2020,23 @@ HttpSysTransaction::InvokeRequestHandler(HttpService& Service, IoBuffer Payload) m_HttpServer.MarkRequest(); + // Track distinct client addresses + { + const SOCKADDR* SockAddr = HttpRequest()->Address.pRemoteAddress; + if (SockAddr->sa_family == AF_INET) + { + const SOCKADDR_IN* V4 = reinterpret_cast<const SOCKADDR_IN*>(SockAddr); + m_HttpServer.MarkClientAddress(&V4->sin_addr, sizeof(V4->sin_addr)); + } + else if (SockAddr->sa_family == AF_INET6) + { + const SOCKADDR_IN6* V6 = reinterpret_cast<const SOCKADDR_IN6*>(SockAddr); + m_HttpServer.MarkClientAddress(&V6->sin6_addr, sizeof(V6->sin6_addr)); + } + } + + m_HttpServer.MarkSessionId(ThisRequest.SessionId()); + // Default request handling # if ZEN_WITH_OTEL |