idle deprovision in hub (#895)

- Feature: Hub watchdog automatically deprovisions inactive provisioned and hibernated instances
- Feature: Added `stats/activity_counters` endpoint to measure server activity
- Feature: Added configuration options for hub watchdog
  - `--hub-watchdog-provisioned-inactivity-timeout-seconds` Inactivity timeout before a provisioned instance is deprovisioned
  - `--hub-watchdog-hibernated-inactivity-timeout-seconds` Inactivity timeout before a hibernated instance is deprovisioned
  - `--hub-watchdog-inactivity-check-margin-seconds` Margin before timeout at which an activity check is issued
  - `--hub-watchdog-cycle-interval-ms` Watchdog poll interval in milliseconds
  - `--hub-watchdog-cycle-processing-budget-ms` Maximum time budget per watchdog cycle in milliseconds
  - `--hub-watchdog-instance-check-throttle-ms` Minimum delay between checks on a single instance
  - `--hub-watchdog-activity-check-connect-timeout-ms` Connect timeout for activity check requests
  - `--hub-watchdog-activity-check-request-timeout-ms` Request timeout for activity check requests

1 files changed, 49 insertions, 26 deletions

diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp
index 493326a32..dec0c3cee 100644
--- a/src/zenserver/storage/objectstore/objectstore.cpp
+++ b/src/zenserver/storage/objectstore/objectstore.cpp
@@ -14,6 +14,7 @@
 #include "zencore/compactbinarybuilder.h"
 #include "zenhttp/httpcommon.h"
 #include "zenhttp/httpserver.h"
+#include "zenhttp/httpstats.h"
 
 #include <filesystem>
 #include <thread>
@@ -220,17 +221,20 @@ private:
 	StringBuilderBase& Builder;
 };
 
-HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg)
-: m_StatusService(StatusService)
+HttpObjectStoreService::HttpObjectStoreService(HttpStatsService& StatsService, HttpStatusService& StatusService, ObjectStoreConfig Cfg)
+: m_StatsService(StatsService)
+, m_StatusService(StatusService)
 , m_Cfg(std::move(Cfg))
 {
 	Inititalize();
+	m_StatsService.RegisterHandler("obj", *this);
 	m_StatusService.RegisterHandler("obj", *this);
 }
 
 HttpObjectStoreService::~HttpObjectStoreService()
 {
 	m_StatusService.UnregisterHandler("obj", *this);
+	m_StatsService.UnregisterHandler("obj", *this);
 }
 
 const char*
@@ -240,8 +244,10 @@ HttpObjectStoreService::BaseUri() const
 }
 
 void
-HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request)
+HttpObjectStoreService::HandleRequest(HttpServerRequest& Request)
 {
+	metrics::OperationTiming::Scope $(m_HttpRequests);
+
 	if (m_Router.HandleRequest(Request) == false)
 	{
 		ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri());
@@ -258,6 +264,23 @@ HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request)
 }
 
 void
+HttpObjectStoreService::HandleStatsRequest(HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+
+	EmitSnapshot("requests", m_HttpRequests, Cbo);
+	Cbo << "total_bytes_served" << m_TotalBytesServed.load();
+
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+uint64_t
+HttpObjectStoreService::GetActivityCounter()
+{
+	return m_HttpRequests.Count();
+}
+
+void
 HttpObjectStoreService::Inititalize()
 {
 	ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize");
@@ -281,27 +304,27 @@ HttpObjectStoreService::Inititalize()
 
 	m_Router.RegisterRoute(
 		"",
-		[this](zen::HttpRouterRequest& Request) { ListBuckets(Request); },
+		[this](HttpRouterRequest& Request) { ListBuckets(Request); },
 		HttpVerb::kGet);
 
 	m_Router.RegisterRoute(
 		"bucket",
-		[this](zen::HttpRouterRequest& Request) { ListBuckets(Request); },
+		[this](HttpRouterRequest& Request) { ListBuckets(Request); },
 		HttpVerb::kGet);
 
 	m_Router.RegisterRoute(
 		"bucket",
-		[this](zen::HttpRouterRequest& Request) { CreateBucket(Request); },
+		[this](HttpRouterRequest& Request) { CreateBucket(Request); },
 		HttpVerb::kPost | HttpVerb::kPut);
 
 	m_Router.RegisterRoute(
 		"bucket",
-		[this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); },
+		[this](HttpRouterRequest& Request) { DeleteBucket(Request); },
 		HttpVerb::kDelete);
 
 	m_Router.RegisterRoute(
 		"bucket/{path}",
-		[this](zen::HttpRouterRequest& Request) {
+		[this](HttpRouterRequest& Request) {
 			const std::string_view Path		= Request.GetCapture(1);
 			const auto			   Sep		= Path.find_last_of('.');
 			const bool			   IsObject = Sep != std::string_view::npos && Path.size() - Sep > 0;
@@ -319,7 +342,7 @@ HttpObjectStoreService::Inititalize()
 
 	m_Router.RegisterRoute(
 		"bucket/{bucket}/{path}",
-		[this](zen::HttpRouterRequest& Request) { PutObject(Request); },
+		[this](HttpRouterRequest& Request) { PutObject(Request); },
 		HttpVerb::kPost | HttpVerb::kPut);
 }
 
@@ -327,7 +350,7 @@ std::filesystem::path
 HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName)
 {
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 
 		if (const auto It = std::find_if(std::begin(m_Cfg.Buckets),
 										 std::end(m_Cfg.Buckets),
@@ -342,7 +365,7 @@ HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName)
 }
 
 void
-HttpObjectStoreService::ListBuckets(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::ListBuckets(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -351,7 +374,7 @@ HttpObjectStoreService::ListBuckets(zen::HttpRouterRequest& Request)
 	CbObjectWriter Response;
 	Response.BeginArray("buckets");
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 
 		// Configured buckets
 		for (const ObjectStoreConfig::BucketConfig& Bucket : m_Cfg.Buckets)
@@ -428,13 +451,13 @@ HttpObjectStoreService::ListBuckets(zen::HttpRouterRequest& Request)
 	}
 	Response.EndArray();
 
-	Response << "total_bytes_served" << TotalBytesServed.load();
+	Response << "total_bytes_served" << m_TotalBytesServed.load();
 
 	return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
 }
 
 void
-HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::CreateBucket(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -448,7 +471,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
 
 	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		if (!IsDir(BucketPath))
 		{
 			CreateDirectories(BucketPath);
@@ -462,7 +485,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
 }
 
 void
-HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path)
+HttpObjectStoreService::ListBucket(HttpRouterRequest& Request, const std::string_view Path)
 {
 	namespace fs = std::filesystem;
 
@@ -533,7 +556,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 
 	if (IsDir(FullPath))
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		Traversal.TraverseFileSystem(FullPath, FileVisitor);
 	}
 	CbObject Result = FileVisitor.GetResult();
@@ -552,7 +575,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 }
 
 void
-HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::DeleteBucket(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -566,7 +589,7 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
 
 	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		DeleteDirectories(BucketPath);
 	}
 
@@ -575,7 +598,7 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
 }
 
 void
-HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path)
+HttpObjectStoreService::GetObject(HttpRouterRequest& Request, const std::string_view Path)
 {
 	namespace fs = std::filesystem;
 
@@ -606,7 +629,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	zen::HttpRanges Ranges;
+	HttpRanges Ranges;
 	if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1)
 	{
 		// Only a single range is supported
@@ -615,7 +638,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 
 	FileContents File;
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		File = ReadFile(FilePath);
 	}
 
@@ -635,7 +658,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 
 	if (Ranges.empty())
 	{
-		const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
+		const uint64_t TotalServed = m_TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
 
 		ZEN_LOG_DEBUG(LogObj,
 					  "GET - '{}/{}' ({}) [OK] (Served: {})",
@@ -650,7 +673,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 	{
 		const auto	   Range	   = Ranges[0];
 		const uint64_t RangeSize   = 1 + (Range.End - Range.Start);
-		const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize;
+		const uint64_t TotalServed = m_TotalBytesServed.fetch_add(RangeSize) + RangeSize;
 
 		ZEN_LOG_DEBUG(LogObj,
 					  "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})",
@@ -674,7 +697,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 }
 
 void
-HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::PutObject(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -699,7 +722,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
 	const fs::path FileDirectory = FilePath.parent_path();
 
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 
 		if (!IsDir(FileDirectory))
 		{