idle deprovision in hub (#895)

- Feature: Hub watchdog automatically deprovisions inactive provisioned and hibernated instances
- Feature: Added `stats/activity_counters` endpoint to measure server activity
- Feature: Added configuration options for hub watchdog
  - `--hub-watchdog-provisioned-inactivity-timeout-seconds` Inactivity timeout before a provisioned instance is deprovisioned
  - `--hub-watchdog-hibernated-inactivity-timeout-seconds` Inactivity timeout before a hibernated instance is deprovisioned
  - `--hub-watchdog-inactivity-check-margin-seconds` Margin before timeout at which an activity check is issued
  - `--hub-watchdog-cycle-interval-ms` Watchdog poll interval in milliseconds
  - `--hub-watchdog-cycle-processing-budget-ms` Maximum time budget per watchdog cycle in milliseconds
  - `--hub-watchdog-instance-check-throttle-ms` Minimum delay between checks on a single instance
  - `--hub-watchdog-activity-check-connect-timeout-ms` Connect timeout for activity check requests
  - `--hub-watchdog-activity-check-request-timeout-ms` Request timeout for activity check requests

1 files changed, 115 insertions, 109 deletions

diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp
index bbdb03ba4..81e244aff 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.cpp
+++ b/src/zenserver/storage/cache/httpstructuredcache.cpp
@@ -1827,113 +1827,12 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::st
 	}
 }
 
-CbObject
-HttpStructuredCacheService::CollectStats()
+void
+HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request)
 {
-	ZEN_MEMSCOPE(GetCacheHttpTag());
-
 	CbObjectWriter Cbo;
-
-	EmitSnapshot("requests", m_HttpRequests, Cbo);
-
-	const uint64_t		 HitCount		  = m_CacheStats.HitCount;
-	const uint64_t		 UpstreamHitCount = m_CacheStats.UpstreamHitCount;
-	const uint64_t		 MissCount		  = m_CacheStats.MissCount;
-	const uint64_t		 WriteCount		  = m_CacheStats.WriteCount;
-	const uint64_t		 BadRequestCount  = m_CacheStats.BadRequestCount;
-	struct CidStoreStats StoreStats		  = m_CidStore.Stats();
-	const uint64_t		 ChunkHitCount	  = StoreStats.HitCount;
-	const uint64_t		 ChunkMissCount	  = StoreStats.MissCount;
-	const uint64_t		 ChunkWriteCount  = StoreStats.WriteCount;
-	const uint64_t		 TotalCount		  = HitCount + MissCount;
-
-	const uint64_t RpcRequests			  = m_CacheStats.RpcRequests;
-	const uint64_t RpcRecordRequests	  = m_CacheStats.RpcRecordRequests;
-	const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
-	const uint64_t RpcValueRequests		  = m_CacheStats.RpcValueRequests;
-	const uint64_t RpcValueBatchRequests  = m_CacheStats.RpcValueBatchRequests;
-	const uint64_t RpcChunkRequests		  = m_CacheStats.RpcChunkRequests;
-	const uint64_t RpcChunkBatchRequests  = m_CacheStats.RpcChunkBatchRequests;
-
-	const CidStoreSize	 CidSize   = m_CidStore.TotalSize();
-	const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
-
-	Cbo.BeginObject("cache");
-	{
-		Cbo << "badrequestcount" << BadRequestCount;
-		Cbo.BeginObject("rpc");
-		Cbo << "count" << RpcRequests;
-		Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests;
-		Cbo.BeginObject("records");
-		Cbo << "count" << RpcRecordRequests;
-		Cbo << "ops" << RpcRecordBatchRequests;
-		Cbo.EndObject();
-		Cbo.BeginObject("values");
-		Cbo << "count" << RpcValueRequests;
-		Cbo << "ops" << RpcValueBatchRequests;
-		Cbo.EndObject();
-		Cbo.BeginObject("chunks");
-		Cbo << "count" << RpcChunkRequests;
-		Cbo << "ops" << RpcChunkBatchRequests;
-		Cbo.EndObject();
-		Cbo.EndObject();
-
-		Cbo.BeginObject("size");
-		{
-			Cbo << "disk" << CacheSize.DiskSize;
-			Cbo << "memory" << CacheSize.MemorySize;
-		}
-		Cbo.EndObject();
-
-		Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
-		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
-
-		if (m_UpstreamCache.IsActive())
-		{
-			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-			Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
-		}
-
-		Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
-
-		{
-			ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats();
-			Cbo.BeginObject("store");
-			Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount
-				<< "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount;
-			const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount;
-			Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0);
-			EmitSnapshot("read", StoreStatsData.GetOps, Cbo);
-			EmitSnapshot("write", StoreStatsData.PutOps, Cbo);
-			Cbo.EndObject();
-		}
-	}
-	Cbo.EndObject();
-
-	if (m_UpstreamCache.IsActive())
-	{
-		EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
-		Cbo.BeginObject("upstream");
-		{
-			m_UpstreamCache.GetStatus(Cbo);
-		}
-		Cbo.EndObject();
-	}
-
-	Cbo.BeginObject("cid");
-	{
-		Cbo.BeginObject("size");
-		{
-			Cbo << "tiny" << CidSize.TinySize;
-			Cbo << "small" << CidSize.SmallSize;
-			Cbo << "large" << CidSize.LargeSize;
-			Cbo << "total" << CidSize.TotalSize;
-		}
-		Cbo.EndObject();
-	}
-	Cbo.EndObject();
-
-	return Cbo.Save();
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
 void
@@ -2156,12 +2055,119 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
-void
-HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request)
+CbObject
+HttpStructuredCacheService::CollectStats()
 {
+	ZEN_MEMSCOPE(GetCacheHttpTag());
+
 	CbObjectWriter Cbo;
-	Cbo << "ok" << true;
-	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+
+	EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+	const uint64_t		 HitCount		  = m_CacheStats.HitCount;
+	const uint64_t		 UpstreamHitCount = m_CacheStats.UpstreamHitCount;
+	const uint64_t		 MissCount		  = m_CacheStats.MissCount;
+	const uint64_t		 WriteCount		  = m_CacheStats.WriteCount;
+	const uint64_t		 BadRequestCount  = m_CacheStats.BadRequestCount;
+	struct CidStoreStats StoreStats		  = m_CidStore.Stats();
+	const uint64_t		 ChunkHitCount	  = StoreStats.HitCount;
+	const uint64_t		 ChunkMissCount	  = StoreStats.MissCount;
+	const uint64_t		 ChunkWriteCount  = StoreStats.WriteCount;
+	const uint64_t		 TotalCount		  = HitCount + MissCount;
+
+	const uint64_t RpcRequests			  = m_CacheStats.RpcRequests;
+	const uint64_t RpcRecordRequests	  = m_CacheStats.RpcRecordRequests;
+	const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
+	const uint64_t RpcValueRequests		  = m_CacheStats.RpcValueRequests;
+	const uint64_t RpcValueBatchRequests  = m_CacheStats.RpcValueBatchRequests;
+	const uint64_t RpcChunkRequests		  = m_CacheStats.RpcChunkRequests;
+	const uint64_t RpcChunkBatchRequests  = m_CacheStats.RpcChunkBatchRequests;
+
+	const CidStoreSize	 CidSize   = m_CidStore.TotalSize();
+	const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
+
+	Cbo.BeginObject("cache");
+	{
+		Cbo << "badrequestcount" << BadRequestCount;
+		Cbo.BeginObject("rpc");
+		Cbo << "count" << RpcRequests;
+		Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests;
+		Cbo.BeginObject("records");
+		Cbo << "count" << RpcRecordRequests;
+		Cbo << "ops" << RpcRecordBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("values");
+		Cbo << "count" << RpcValueRequests;
+		Cbo << "ops" << RpcValueBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("chunks");
+		Cbo << "count" << RpcChunkRequests;
+		Cbo << "ops" << RpcChunkBatchRequests;
+		Cbo.EndObject();
+		Cbo.EndObject();
+
+		Cbo.BeginObject("size");
+		{
+			Cbo << "disk" << CacheSize.DiskSize;
+			Cbo << "memory" << CacheSize.MemorySize;
+		}
+		Cbo.EndObject();
+
+		Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
+		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
+
+		if (m_UpstreamCache.IsActive())
+		{
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+			Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+		}
+
+		Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
+
+		{
+			ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats();
+			Cbo.BeginObject("store");
+			Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount
+				<< "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount;
+			const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount;
+			Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0);
+			EmitSnapshot("read", StoreStatsData.GetOps, Cbo);
+			EmitSnapshot("write", StoreStatsData.PutOps, Cbo);
+			Cbo.EndObject();
+		}
+	}
+	Cbo.EndObject();
+
+	if (m_UpstreamCache.IsActive())
+	{
+		EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
+		Cbo.BeginObject("upstream");
+		{
+			m_UpstreamCache.GetStatus(Cbo);
+		}
+		Cbo.EndObject();
+	}
+
+	Cbo.BeginObject("cid");
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "tiny" << CidSize.TinySize;
+			Cbo << "small" << CidSize.SmallSize;
+			Cbo << "large" << CidSize.LargeSize;
+			Cbo << "total" << CidSize.TotalSize;
+		}
+		Cbo.EndObject();
+	}
+	Cbo.EndObject();
+
+	return Cbo.Save();
+}
+
+uint64_t
+HttpStructuredCacheService::GetActivityCounter()
+{
+	return m_HttpRequests.Count();
 }
 
 bool