From 776d76d299748a79b9cb25593cd8266cb26a6553 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 27 Mar 2026 11:13:02 +0100 Subject: idle deprovision in hub (#895) - Feature: Hub watchdog automatically deprovisions inactive provisioned and hibernated instances - Feature: Added `stats/activity_counters` endpoint to measure server activity - Feature: Added configuration options for hub watchdog - `--hub-watchdog-provisioned-inactivity-timeout-seconds` Inactivity timeout before a provisioned instance is deprovisioned - `--hub-watchdog-hibernated-inactivity-timeout-seconds` Inactivity timeout before a hibernated instance is deprovisioned - `--hub-watchdog-inactivity-check-margin-seconds` Margin before timeout at which an activity check is issued - `--hub-watchdog-cycle-interval-ms` Watchdog poll interval in milliseconds - `--hub-watchdog-cycle-processing-budget-ms` Maximum time budget per watchdog cycle in milliseconds - `--hub-watchdog-instance-check-throttle-ms` Minimum delay between checks on a single instance - `--hub-watchdog-activity-check-connect-timeout-ms` Connect timeout for activity check requests - `--hub-watchdog-activity-check-request-timeout-ms` Request timeout for activity check requests --- .../storage/cache/httpstructuredcache.cpp | 224 +++++++++++---------- 1 file changed, 115 insertions(+), 109 deletions(-) (limited to 'src/zenserver/storage/cache/httpstructuredcache.cpp') diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp index bbdb03ba4..81e244aff 100644 --- a/src/zenserver/storage/cache/httpstructuredcache.cpp +++ b/src/zenserver/storage/cache/httpstructuredcache.cpp @@ -1827,113 +1827,12 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::st } } -CbObject -HttpStructuredCacheService::CollectStats() +void +HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request) { - ZEN_MEMSCOPE(GetCacheHttpTag()); - CbObjectWriter Cbo; - - EmitSnapshot("requests", m_HttpRequests, Cbo); - - const uint64_t HitCount = m_CacheStats.HitCount; - const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; - const uint64_t MissCount = m_CacheStats.MissCount; - const uint64_t WriteCount = m_CacheStats.WriteCount; - const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; - struct CidStoreStats StoreStats = m_CidStore.Stats(); - const uint64_t ChunkHitCount = StoreStats.HitCount; - const uint64_t ChunkMissCount = StoreStats.MissCount; - const uint64_t ChunkWriteCount = StoreStats.WriteCount; - const uint64_t TotalCount = HitCount + MissCount; - - const uint64_t RpcRequests = m_CacheStats.RpcRequests; - const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests; - const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests; - const uint64_t RpcValueRequests = m_CacheStats.RpcValueRequests; - const uint64_t RpcValueBatchRequests = m_CacheStats.RpcValueBatchRequests; - const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests; - const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests; - - const CidStoreSize CidSize = m_CidStore.TotalSize(); - const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); - - Cbo.BeginObject("cache"); - { - Cbo << "badrequestcount" << BadRequestCount; - Cbo.BeginObject("rpc"); - Cbo << "count" << RpcRequests; - Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests; - Cbo.BeginObject("records"); - Cbo << "count" << RpcRecordRequests; - Cbo << "ops" << RpcRecordBatchRequests; - Cbo.EndObject(); - Cbo.BeginObject("values"); - Cbo << "count" << RpcValueRequests; - Cbo << "ops" << RpcValueBatchRequests; - Cbo.EndObject(); - Cbo.BeginObject("chunks"); - Cbo << "count" << RpcChunkRequests; - Cbo << "ops" << RpcChunkBatchRequests; - Cbo.EndObject(); - Cbo.EndObject(); - - Cbo.BeginObject("size"); - { - Cbo << "disk" << CacheSize.DiskSize; - Cbo << "memory" << CacheSize.MemorySize; - } - Cbo.EndObject(); - - Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount; - Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0); - - if (m_UpstreamCache.IsActive()) - { - Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); - Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount; - } - - Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; - - { - ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats(); - Cbo.BeginObject("store"); - Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount - << "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount; - const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount; - Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0); - EmitSnapshot("read", StoreStatsData.GetOps, Cbo); - EmitSnapshot("write", StoreStatsData.PutOps, Cbo); - Cbo.EndObject(); - } - } - Cbo.EndObject(); - - if (m_UpstreamCache.IsActive()) - { - EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo); - Cbo.BeginObject("upstream"); - { - m_UpstreamCache.GetStatus(Cbo); - } - Cbo.EndObject(); - } - - Cbo.BeginObject("cid"); - { - Cbo.BeginObject("size"); - { - Cbo << "tiny" << CidSize.TinySize; - Cbo << "small" << CidSize.SmallSize; - Cbo << "large" << CidSize.LargeSize; - Cbo << "total" << CidSize.TotalSize; - } - Cbo.EndObject(); - } - Cbo.EndObject(); - - return Cbo.Save(); + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } void @@ -2156,12 +2055,119 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } -void -HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request) +CbObject +HttpStructuredCacheService::CollectStats() { + ZEN_MEMSCOPE(GetCacheHttpTag()); + CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + const uint64_t HitCount = m_CacheStats.HitCount; + const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; + const uint64_t MissCount = m_CacheStats.MissCount; + const uint64_t WriteCount = m_CacheStats.WriteCount; + const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; + struct CidStoreStats StoreStats = m_CidStore.Stats(); + const uint64_t ChunkHitCount = StoreStats.HitCount; + const uint64_t ChunkMissCount = StoreStats.MissCount; + const uint64_t ChunkWriteCount = StoreStats.WriteCount; + const uint64_t TotalCount = HitCount + MissCount; + + const uint64_t RpcRequests = m_CacheStats.RpcRequests; + const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests; + const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests; + const uint64_t RpcValueRequests = m_CacheStats.RpcValueRequests; + const uint64_t RpcValueBatchRequests = m_CacheStats.RpcValueBatchRequests; + const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests; + const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests; + + const CidStoreSize CidSize = m_CidStore.TotalSize(); + const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); + + Cbo.BeginObject("cache"); + { + Cbo << "badrequestcount" << BadRequestCount; + Cbo.BeginObject("rpc"); + Cbo << "count" << RpcRequests; + Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests; + Cbo.BeginObject("records"); + Cbo << "count" << RpcRecordRequests; + Cbo << "ops" << RpcRecordBatchRequests; + Cbo.EndObject(); + Cbo.BeginObject("values"); + Cbo << "count" << RpcValueRequests; + Cbo << "ops" << RpcValueBatchRequests; + Cbo.EndObject(); + Cbo.BeginObject("chunks"); + Cbo << "count" << RpcChunkRequests; + Cbo << "ops" << RpcChunkBatchRequests; + Cbo.EndObject(); + Cbo.EndObject(); + + Cbo.BeginObject("size"); + { + Cbo << "disk" << CacheSize.DiskSize; + Cbo << "memory" << CacheSize.MemorySize; + } + Cbo.EndObject(); + + Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount; + Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0); + + if (m_UpstreamCache.IsActive()) + { + Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); + Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount; + } + + Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; + + { + ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats(); + Cbo.BeginObject("store"); + Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount + << "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount; + const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount; + Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0); + EmitSnapshot("read", StoreStatsData.GetOps, Cbo); + EmitSnapshot("write", StoreStatsData.PutOps, Cbo); + Cbo.EndObject(); + } + } + Cbo.EndObject(); + + if (m_UpstreamCache.IsActive()) + { + EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo); + Cbo.BeginObject("upstream"); + { + m_UpstreamCache.GetStatus(Cbo); + } + Cbo.EndObject(); + } + + Cbo.BeginObject("cid"); + { + Cbo.BeginObject("size"); + { + Cbo << "tiny" << CidSize.TinySize; + Cbo << "small" << CidSize.SmallSize; + Cbo << "large" << CidSize.LargeSize; + Cbo << "total" << CidSize.TotalSize; + } + Cbo.EndObject(); + } + Cbo.EndObject(); + + return Cbo.Save(); +} + +uint64_t +HttpStructuredCacheService::GetActivityCounter() +{ + return m_HttpRequests.Count(); } bool -- cgit v1.2.3