idle deprovision in hub (#895)

- Feature: Hub watchdog automatically deprovisions inactive provisioned and hibernated instances
- Feature: Added `stats/activity_counters` endpoint to measure server activity
- Feature: Added configuration options for hub watchdog
  - `--hub-watchdog-provisioned-inactivity-timeout-seconds` Inactivity timeout before a provisioned instance is deprovisioned
  - `--hub-watchdog-hibernated-inactivity-timeout-seconds` Inactivity timeout before a hibernated instance is deprovisioned
  - `--hub-watchdog-inactivity-check-margin-seconds` Margin before timeout at which an activity check is issued
  - `--hub-watchdog-cycle-interval-ms` Watchdog poll interval in milliseconds
  - `--hub-watchdog-cycle-processing-budget-ms` Maximum time budget per watchdog cycle in milliseconds
  - `--hub-watchdog-instance-check-throttle-ms` Minimum delay between checks on a single instance
  - `--hub-watchdog-activity-check-connect-timeout-ms` Connect timeout for activity check requests
  - `--hub-watchdog-activity-check-request-timeout-ms` Request timeout for activity check requests

13 files changed, 263 insertions, 207 deletions

diff --git a/src/zenserver/storage/admin/admin.h b/src/zenserver/storage/admin/admin.h
index ee3da4579..361153e42 100644
--- a/src/zenserver/storage/admin/admin.h
+++ b/src/zenserver/storage/admin/admin.h
@@ -13,7 +13,7 @@ class JobQueue;
 class ZenCacheStore;
 struct ZenServerConfig;
 
-class HttpAdminService : public zen::HttpService
+class HttpAdminService : public HttpService
 {
 public:
 	struct LogPaths
@@ -31,7 +31,7 @@ public:
 	~HttpAdminService();
 
 	virtual const char* BaseUri() const override;
-	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
 private:
 	HttpRequestRouter	   m_Router;
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index de9589078..bbbb0c37b 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -605,6 +605,26 @@ HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req)
 	return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject);
 }
 
+void
+HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Cbo.BeginObject("capabilities");
+	{
+		Cbo << "maxrangecountperrequest" << MaxRangeCountPerRequestSupported;
+	}
+	Cbo.EndObject();  // capabilities
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
+HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
+{
+	Request.WriteResponse(HttpResponseCode::OK, CollectStats());
+}
+
 CbObject
 HttpBuildStoreService::CollectStats()
 {
@@ -663,24 +683,10 @@ HttpBuildStoreService::CollectStats()
 	return Cbo.Save();
 }
 
-void
-HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
-{
-	Request.WriteResponse(HttpResponseCode::OK, CollectStats());
-}
-
-void
-HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request)
+uint64_t
+HttpBuildStoreService::GetActivityCounter()
 {
-	ZEN_TRACE_CPU("HttpBuildStoreService::Status");
-	CbObjectWriter Cbo;
-	Cbo << "ok" << true;
-	Cbo.BeginObject("capabilities");
-	{
-		Cbo << "maxrangecountperrequest" << MaxRangeCountPerRequestSupported;
-	}
-	Cbo.EndObject();  // capabilities
-	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+	return m_HttpRequests.Count();
 }
 
 }  // namespace zen
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h
index 2a09b71cf..864d12edc 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.h
+++ b/src/zenserver/storage/buildstore/httpbuildstore.h
@@ -13,18 +13,19 @@ namespace zen {
 
 class BuildStore;
 
-class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider
+class HttpBuildStoreService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
 {
 public:
 	HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store);
 	virtual ~HttpBuildStoreService();
 
 	virtual const char* BaseUri() const override;
-	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
-	virtual CbObject CollectStats() override;
-	virtual void	 HandleStatsRequest(HttpServerRequest& Request) override;
 	virtual void	 HandleStatusRequest(HttpServerRequest& Request) override;
+	virtual void	 HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual CbObject CollectStats() override;
+	virtual uint64_t GetActivityCounter() override;
 
 private:
 	struct BuildStoreStats
diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp
index bbdb03ba4..81e244aff 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.cpp
+++ b/src/zenserver/storage/cache/httpstructuredcache.cpp
@@ -1827,113 +1827,12 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::st
 	}
 }
 
-CbObject
-HttpStructuredCacheService::CollectStats()
+void
+HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request)
 {
-	ZEN_MEMSCOPE(GetCacheHttpTag());
-
 	CbObjectWriter Cbo;
-
-	EmitSnapshot("requests", m_HttpRequests, Cbo);
-
-	const uint64_t		 HitCount		  = m_CacheStats.HitCount;
-	const uint64_t		 UpstreamHitCount = m_CacheStats.UpstreamHitCount;
-	const uint64_t		 MissCount		  = m_CacheStats.MissCount;
-	const uint64_t		 WriteCount		  = m_CacheStats.WriteCount;
-	const uint64_t		 BadRequestCount  = m_CacheStats.BadRequestCount;
-	struct CidStoreStats StoreStats		  = m_CidStore.Stats();
-	const uint64_t		 ChunkHitCount	  = StoreStats.HitCount;
-	const uint64_t		 ChunkMissCount	  = StoreStats.MissCount;
-	const uint64_t		 ChunkWriteCount  = StoreStats.WriteCount;
-	const uint64_t		 TotalCount		  = HitCount + MissCount;
-
-	const uint64_t RpcRequests			  = m_CacheStats.RpcRequests;
-	const uint64_t RpcRecordRequests	  = m_CacheStats.RpcRecordRequests;
-	const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
-	const uint64_t RpcValueRequests		  = m_CacheStats.RpcValueRequests;
-	const uint64_t RpcValueBatchRequests  = m_CacheStats.RpcValueBatchRequests;
-	const uint64_t RpcChunkRequests		  = m_CacheStats.RpcChunkRequests;
-	const uint64_t RpcChunkBatchRequests  = m_CacheStats.RpcChunkBatchRequests;
-
-	const CidStoreSize	 CidSize   = m_CidStore.TotalSize();
-	const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
-
-	Cbo.BeginObject("cache");
-	{
-		Cbo << "badrequestcount" << BadRequestCount;
-		Cbo.BeginObject("rpc");
-		Cbo << "count" << RpcRequests;
-		Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests;
-		Cbo.BeginObject("records");
-		Cbo << "count" << RpcRecordRequests;
-		Cbo << "ops" << RpcRecordBatchRequests;
-		Cbo.EndObject();
-		Cbo.BeginObject("values");
-		Cbo << "count" << RpcValueRequests;
-		Cbo << "ops" << RpcValueBatchRequests;
-		Cbo.EndObject();
-		Cbo.BeginObject("chunks");
-		Cbo << "count" << RpcChunkRequests;
-		Cbo << "ops" << RpcChunkBatchRequests;
-		Cbo.EndObject();
-		Cbo.EndObject();
-
-		Cbo.BeginObject("size");
-		{
-			Cbo << "disk" << CacheSize.DiskSize;
-			Cbo << "memory" << CacheSize.MemorySize;
-		}
-		Cbo.EndObject();
-
-		Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
-		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
-
-		if (m_UpstreamCache.IsActive())
-		{
-			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-			Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
-		}
-
-		Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
-
-		{
-			ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats();
-			Cbo.BeginObject("store");
-			Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount
-				<< "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount;
-			const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount;
-			Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0);
-			EmitSnapshot("read", StoreStatsData.GetOps, Cbo);
-			EmitSnapshot("write", StoreStatsData.PutOps, Cbo);
-			Cbo.EndObject();
-		}
-	}
-	Cbo.EndObject();
-
-	if (m_UpstreamCache.IsActive())
-	{
-		EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
-		Cbo.BeginObject("upstream");
-		{
-			m_UpstreamCache.GetStatus(Cbo);
-		}
-		Cbo.EndObject();
-	}
-
-	Cbo.BeginObject("cid");
-	{
-		Cbo.BeginObject("size");
-		{
-			Cbo << "tiny" << CidSize.TinySize;
-			Cbo << "small" << CidSize.SmallSize;
-			Cbo << "large" << CidSize.LargeSize;
-			Cbo << "total" << CidSize.TotalSize;
-		}
-		Cbo.EndObject();
-	}
-	Cbo.EndObject();
-
-	return Cbo.Save();
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
 void
@@ -2156,12 +2055,119 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
-void
-HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request)
+CbObject
+HttpStructuredCacheService::CollectStats()
 {
+	ZEN_MEMSCOPE(GetCacheHttpTag());
+
 	CbObjectWriter Cbo;
-	Cbo << "ok" << true;
-	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+
+	EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+	const uint64_t		 HitCount		  = m_CacheStats.HitCount;
+	const uint64_t		 UpstreamHitCount = m_CacheStats.UpstreamHitCount;
+	const uint64_t		 MissCount		  = m_CacheStats.MissCount;
+	const uint64_t		 WriteCount		  = m_CacheStats.WriteCount;
+	const uint64_t		 BadRequestCount  = m_CacheStats.BadRequestCount;
+	struct CidStoreStats StoreStats		  = m_CidStore.Stats();
+	const uint64_t		 ChunkHitCount	  = StoreStats.HitCount;
+	const uint64_t		 ChunkMissCount	  = StoreStats.MissCount;
+	const uint64_t		 ChunkWriteCount  = StoreStats.WriteCount;
+	const uint64_t		 TotalCount		  = HitCount + MissCount;
+
+	const uint64_t RpcRequests			  = m_CacheStats.RpcRequests;
+	const uint64_t RpcRecordRequests	  = m_CacheStats.RpcRecordRequests;
+	const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
+	const uint64_t RpcValueRequests		  = m_CacheStats.RpcValueRequests;
+	const uint64_t RpcValueBatchRequests  = m_CacheStats.RpcValueBatchRequests;
+	const uint64_t RpcChunkRequests		  = m_CacheStats.RpcChunkRequests;
+	const uint64_t RpcChunkBatchRequests  = m_CacheStats.RpcChunkBatchRequests;
+
+	const CidStoreSize	 CidSize   = m_CidStore.TotalSize();
+	const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
+
+	Cbo.BeginObject("cache");
+	{
+		Cbo << "badrequestcount" << BadRequestCount;
+		Cbo.BeginObject("rpc");
+		Cbo << "count" << RpcRequests;
+		Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests;
+		Cbo.BeginObject("records");
+		Cbo << "count" << RpcRecordRequests;
+		Cbo << "ops" << RpcRecordBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("values");
+		Cbo << "count" << RpcValueRequests;
+		Cbo << "ops" << RpcValueBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("chunks");
+		Cbo << "count" << RpcChunkRequests;
+		Cbo << "ops" << RpcChunkBatchRequests;
+		Cbo.EndObject();
+		Cbo.EndObject();
+
+		Cbo.BeginObject("size");
+		{
+			Cbo << "disk" << CacheSize.DiskSize;
+			Cbo << "memory" << CacheSize.MemorySize;
+		}
+		Cbo.EndObject();
+
+		Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
+		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
+
+		if (m_UpstreamCache.IsActive())
+		{
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+			Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+		}
+
+		Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
+
+		{
+			ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats();
+			Cbo.BeginObject("store");
+			Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount
+				<< "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount;
+			const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount;
+			Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0);
+			EmitSnapshot("read", StoreStatsData.GetOps, Cbo);
+			EmitSnapshot("write", StoreStatsData.PutOps, Cbo);
+			Cbo.EndObject();
+		}
+	}
+	Cbo.EndObject();
+
+	if (m_UpstreamCache.IsActive())
+	{
+		EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
+		Cbo.BeginObject("upstream");
+		{
+			m_UpstreamCache.GetStatus(Cbo);
+		}
+		Cbo.EndObject();
+	}
+
+	Cbo.BeginObject("cid");
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "tiny" << CidSize.TinySize;
+			Cbo << "small" << CidSize.SmallSize;
+			Cbo << "large" << CidSize.LargeSize;
+			Cbo << "total" << CidSize.TotalSize;
+		}
+		Cbo.EndObject();
+	}
+	Cbo.EndObject();
+
+	return Cbo.Save();
+}
+
+uint64_t
+HttpStructuredCacheService::GetActivityCounter()
+{
+	return m_HttpRequests.Count();
 }
 
 bool
diff --git a/src/zenserver/storage/cache/httpstructuredcache.h b/src/zenserver/storage/cache/httpstructuredcache.h
index d462415d4..fc80b449e 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.h
+++ b/src/zenserver/storage/cache/httpstructuredcache.h
@@ -105,9 +105,10 @@ private:
 	void			 HandleCacheRequest(HttpServerRequest& Request);
 	void			 HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view Namespace);
 	void			 HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket);
-	virtual CbObject CollectStats() override;
-	virtual void	 HandleStatsRequest(HttpServerRequest& Request) override;
 	virtual void	 HandleStatusRequest(HttpServerRequest& Request) override;
+	virtual void	 HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual CbObject CollectStats() override;
+	virtual uint64_t GetActivityCounter() override;
 
 	bool AreDiskWritesAllowed() const;
 
diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp
index 493326a32..dec0c3cee 100644
--- a/src/zenserver/storage/objectstore/objectstore.cpp
+++ b/src/zenserver/storage/objectstore/objectstore.cpp
@@ -14,6 +14,7 @@
 #include "zencore/compactbinarybuilder.h"
 #include "zenhttp/httpcommon.h"
 #include "zenhttp/httpserver.h"
+#include "zenhttp/httpstats.h"
 
 #include <filesystem>
 #include <thread>
@@ -220,17 +221,20 @@ private:
 	StringBuilderBase& Builder;
 };
 
-HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg)
-: m_StatusService(StatusService)
+HttpObjectStoreService::HttpObjectStoreService(HttpStatsService& StatsService, HttpStatusService& StatusService, ObjectStoreConfig Cfg)
+: m_StatsService(StatsService)
+, m_StatusService(StatusService)
 , m_Cfg(std::move(Cfg))
 {
 	Inititalize();
+	m_StatsService.RegisterHandler("obj", *this);
 	m_StatusService.RegisterHandler("obj", *this);
 }
 
 HttpObjectStoreService::~HttpObjectStoreService()
 {
 	m_StatusService.UnregisterHandler("obj", *this);
+	m_StatsService.UnregisterHandler("obj", *this);
 }
 
 const char*
@@ -240,8 +244,10 @@ HttpObjectStoreService::BaseUri() const
 }
 
 void
-HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request)
+HttpObjectStoreService::HandleRequest(HttpServerRequest& Request)
 {
+	metrics::OperationTiming::Scope $(m_HttpRequests);
+
 	if (m_Router.HandleRequest(Request) == false)
 	{
 		ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri());
@@ -258,6 +264,23 @@ HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request)
 }
 
 void
+HttpObjectStoreService::HandleStatsRequest(HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+
+	EmitSnapshot("requests", m_HttpRequests, Cbo);
+	Cbo << "total_bytes_served" << m_TotalBytesServed.load();
+
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+uint64_t
+HttpObjectStoreService::GetActivityCounter()
+{
+	return m_HttpRequests.Count();
+}
+
+void
 HttpObjectStoreService::Inititalize()
 {
 	ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize");
@@ -281,27 +304,27 @@ HttpObjectStoreService::Inititalize()
 
 	m_Router.RegisterRoute(
 		"",
-		[this](zen::HttpRouterRequest& Request) { ListBuckets(Request); },
+		[this](HttpRouterRequest& Request) { ListBuckets(Request); },
 		HttpVerb::kGet);
 
 	m_Router.RegisterRoute(
 		"bucket",
-		[this](zen::HttpRouterRequest& Request) { ListBuckets(Request); },
+		[this](HttpRouterRequest& Request) { ListBuckets(Request); },
 		HttpVerb::kGet);
 
 	m_Router.RegisterRoute(
 		"bucket",
-		[this](zen::HttpRouterRequest& Request) { CreateBucket(Request); },
+		[this](HttpRouterRequest& Request) { CreateBucket(Request); },
 		HttpVerb::kPost | HttpVerb::kPut);
 
 	m_Router.RegisterRoute(
 		"bucket",
-		[this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); },
+		[this](HttpRouterRequest& Request) { DeleteBucket(Request); },
 		HttpVerb::kDelete);
 
 	m_Router.RegisterRoute(
 		"bucket/{path}",
-		[this](zen::HttpRouterRequest& Request) {
+		[this](HttpRouterRequest& Request) {
 			const std::string_view Path		= Request.GetCapture(1);
 			const auto			   Sep		= Path.find_last_of('.');
 			const bool			   IsObject = Sep != std::string_view::npos && Path.size() - Sep > 0;
@@ -319,7 +342,7 @@ HttpObjectStoreService::Inititalize()
 
 	m_Router.RegisterRoute(
 		"bucket/{bucket}/{path}",
-		[this](zen::HttpRouterRequest& Request) { PutObject(Request); },
+		[this](HttpRouterRequest& Request) { PutObject(Request); },
 		HttpVerb::kPost | HttpVerb::kPut);
 }
 
@@ -327,7 +350,7 @@ std::filesystem::path
 HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName)
 {
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 
 		if (const auto It = std::find_if(std::begin(m_Cfg.Buckets),
 										 std::end(m_Cfg.Buckets),
@@ -342,7 +365,7 @@ HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName)
 }
 
 void
-HttpObjectStoreService::ListBuckets(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::ListBuckets(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -351,7 +374,7 @@ HttpObjectStoreService::ListBuckets(zen::HttpRouterRequest& Request)
 	CbObjectWriter Response;
 	Response.BeginArray("buckets");
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 
 		// Configured buckets
 		for (const ObjectStoreConfig::BucketConfig& Bucket : m_Cfg.Buckets)
@@ -428,13 +451,13 @@ HttpObjectStoreService::ListBuckets(zen::HttpRouterRequest& Request)
 	}
 	Response.EndArray();
 
-	Response << "total_bytes_served" << TotalBytesServed.load();
+	Response << "total_bytes_served" << m_TotalBytesServed.load();
 
 	return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
 }
 
 void
-HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::CreateBucket(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -448,7 +471,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
 
 	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		if (!IsDir(BucketPath))
 		{
 			CreateDirectories(BucketPath);
@@ -462,7 +485,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
 }
 
 void
-HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path)
+HttpObjectStoreService::ListBucket(HttpRouterRequest& Request, const std::string_view Path)
 {
 	namespace fs = std::filesystem;
 
@@ -533,7 +556,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 
 	if (IsDir(FullPath))
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		Traversal.TraverseFileSystem(FullPath, FileVisitor);
 	}
 	CbObject Result = FileVisitor.GetResult();
@@ -552,7 +575,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 }
 
 void
-HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::DeleteBucket(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -566,7 +589,7 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
 
 	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		DeleteDirectories(BucketPath);
 	}
 
@@ -575,7 +598,7 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
 }
 
 void
-HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path)
+HttpObjectStoreService::GetObject(HttpRouterRequest& Request, const std::string_view Path)
 {
 	namespace fs = std::filesystem;
 
@@ -606,7 +629,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	zen::HttpRanges Ranges;
+	HttpRanges Ranges;
 	if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1)
 	{
 		// Only a single range is supported
@@ -615,7 +638,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 
 	FileContents File;
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		File = ReadFile(FilePath);
 	}
 
@@ -635,7 +658,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 
 	if (Ranges.empty())
 	{
-		const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
+		const uint64_t TotalServed = m_TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
 
 		ZEN_LOG_DEBUG(LogObj,
 					  "GET - '{}/{}' ({}) [OK] (Served: {})",
@@ -650,7 +673,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 	{
 		const auto	   Range	   = Ranges[0];
 		const uint64_t RangeSize   = 1 + (Range.End - Range.Start);
-		const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize;
+		const uint64_t TotalServed = m_TotalBytesServed.fetch_add(RangeSize) + RangeSize;
 
 		ZEN_LOG_DEBUG(LogObj,
 					  "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})",
@@ -674,7 +697,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 }
 
 void
-HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::PutObject(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -699,7 +722,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
 	const fs::path FileDirectory = FilePath.parent_path();
 
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 
 		if (!IsDir(FileDirectory))
 		{
diff --git a/src/zenserver/storage/objectstore/objectstore.h b/src/zenserver/storage/objectstore/objectstore.h
index cc47b50c4..8a25e750b 100644
--- a/src/zenserver/storage/objectstore/objectstore.h
+++ b/src/zenserver/storage/objectstore/objectstore.h
@@ -11,6 +11,7 @@
 namespace zen {
 
 class HttpRouterRequest;
+class HttpStatsService;
 
 struct ObjectStoreConfig
 {
@@ -24,31 +25,35 @@ struct ObjectStoreConfig
 	std::vector<BucketConfig> Buckets;
 };
 
-class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider
+class HttpObjectStoreService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
 {
 public:
-	HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg);
+	HttpObjectStoreService(HttpStatsService& StatsService, HttpStatusService& StatusService, ObjectStoreConfig Cfg);
 	virtual ~HttpObjectStoreService();
 
 	virtual const char* BaseUri() const override;
-	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleRequest(HttpServerRequest& Request) override;
 	virtual void		HandleStatusRequest(HttpServerRequest& Request) override;
+	virtual void		HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual uint64_t	GetActivityCounter() override;
 
 private:
 	void				  Inititalize();
 	std::filesystem::path GetBucketDirectory(std::string_view BucketName);
-	void				  ListBuckets(zen::HttpRouterRequest& Request);
-	void				  CreateBucket(zen::HttpRouterRequest& Request);
-	void				  ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path);
-	void				  DeleteBucket(zen::HttpRouterRequest& Request);
-	void				  GetObject(zen::HttpRouterRequest& Request, const std::string_view Path);
-	void				  PutObject(zen::HttpRouterRequest& Request);
-
-	HttpStatusService&	 m_StatusService;
-	ObjectStoreConfig	 m_Cfg;
-	std::mutex			 BucketsMutex;
-	HttpRequestRouter	 m_Router;
-	std::atomic_uint64_t TotalBytesServed{0};
+	void				  ListBuckets(HttpRouterRequest& Request);
+	void				  CreateBucket(HttpRouterRequest& Request);
+	void				  ListBucket(HttpRouterRequest& Request, const std::string_view Path);
+	void				  DeleteBucket(HttpRouterRequest& Request);
+	void				  GetObject(HttpRouterRequest& Request, const std::string_view Path);
+	void				  PutObject(HttpRouterRequest& Request);
+
+	HttpStatsService&		 m_StatsService;
+	HttpStatusService&		 m_StatusService;
+	ObjectStoreConfig		 m_Cfg;
+	std::mutex				 m_BucketsMutex;
+	HttpRequestRouter		 m_Router;
+	std::atomic_uint64_t	 m_TotalBytesServed{0};
+	metrics::OperationTiming m_HttpRequests;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index 03b8aa382..9d132a22a 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -836,6 +836,21 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
 	}
 }
 
+void
+HttpProjectService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpProjectService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
+HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+	HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
+}
+
 CbObject
 HttpProjectService::CollectStats()
 {
@@ -906,19 +921,10 @@ HttpProjectService::CollectStats()
 	return Cbo.Save();
 }
 
-void
-HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+uint64_t
+HttpProjectService::GetActivityCounter()
 {
-	HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
-}
-
-void
-HttpProjectService::HandleStatusRequest(HttpServerRequest& Request)
-{
-	ZEN_TRACE_CPU("HttpProjectService::Status");
-	CbObjectWriter Cbo;
-	Cbo << "ok" << true;
-	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+	return m_HttpRequests.Count();
 }
 
 void
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h
index 917337324..e3ed02f26 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.h
+++ b/src/zenserver/storage/projectstore/httpprojectstore.h
@@ -53,9 +53,10 @@ public:
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
-	virtual CbObject CollectStats() override;
-	virtual void	 HandleStatsRequest(HttpServerRequest& Request) override;
 	virtual void	 HandleStatusRequest(HttpServerRequest& Request) override;
+	virtual void	 HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual CbObject CollectStats() override;
+	virtual uint64_t GetActivityCounter() override;
 
 private:
 	struct ProjectStats
diff --git a/src/zenserver/storage/upstream/upstreamservice.h b/src/zenserver/storage/upstream/upstreamservice.h
index f1da03c8c..c0063c055 100644
--- a/src/zenserver/storage/upstream/upstreamservice.h
+++ b/src/zenserver/storage/upstream/upstreamservice.h
@@ -9,14 +9,14 @@ namespace zen {
 class AuthMgr;
 class UpstreamCache;
 
-class HttpUpstreamService final : public zen::HttpService
+class HttpUpstreamService final : public HttpService
 {
 public:
 	HttpUpstreamService(UpstreamCache& Upstream, AuthMgr& Mgr);
 	virtual ~HttpUpstreamService();
 
 	virtual const char* BaseUri() const override;
-	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
 private:
 	UpstreamCache&	  m_Upstream;
diff --git a/src/zenserver/storage/workspaces/httpworkspaces.cpp b/src/zenserver/storage/workspaces/httpworkspaces.cpp
index 785dd62f0..0119912e9 100644
--- a/src/zenserver/storage/workspaces/httpworkspaces.cpp
+++ b/src/zenserver/storage/workspaces/httpworkspaces.cpp
@@ -110,6 +110,21 @@ HttpWorkspacesService::HandleRequest(HttpServerRequest& Request)
 	}
 }
 
+void
+HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpWorkspacesService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
+HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+	HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
+}
+
 CbObject
 HttpWorkspacesService::CollectStats()
 {
@@ -153,19 +168,10 @@ HttpWorkspacesService::CollectStats()
 	return Cbo.Save();
 }
 
-void
-HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
+uint64_t
+HttpWorkspacesService::GetActivityCounter()
 {
-	HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
-}
-
-void
-HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request)
-{
-	ZEN_TRACE_CPU("HttpWorkspacesService::Status");
-	CbObjectWriter Cbo;
-	Cbo << "ok" << true;
-	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+	return m_HttpRequests.Count();
 }
 
 void
diff --git a/src/zenserver/storage/workspaces/httpworkspaces.h b/src/zenserver/storage/workspaces/httpworkspaces.h
index 7c5ddeff1..4af1316f8 100644
--- a/src/zenserver/storage/workspaces/httpworkspaces.h
+++ b/src/zenserver/storage/workspaces/httpworkspaces.h
@@ -29,9 +29,10 @@ public:
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
-	virtual CbObject CollectStats() override;
-	virtual void	 HandleStatsRequest(HttpServerRequest& Request) override;
 	virtual void	 HandleStatusRequest(HttpServerRequest& Request) override;
+	virtual void	 HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual CbObject CollectStats() override;
+	virtual uint64_t GetActivityCounter() override;
 
 private:
 	struct WorkspacesStats
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index de00eb1c2..bc0a8f4ac 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -170,7 +170,7 @@ ZenStorageServer::RegisterServices()
 		m_Http->RegisterService(*m_HttpSessionsService);
 	}
 
-	m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
+	m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatsService, m_StatusService);
 
 	if (m_FrontendService)
 	{
@@ -307,7 +307,7 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
 			ObjCfg.Buckets.push_back(std::move(NewBucket));
 		}
 
-		m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatusService, std::move(ObjCfg));
+		m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatsService, m_StatusService, std::move(ObjCfg));
 	}
 
 	if (ServerOptions.BuildStoreConfig.Enabled)