idle deprovision in hub (#895)

- Feature: Hub watchdog automatically deprovisions inactive provisioned and hibernated instances
- Feature: Added `stats/activity_counters` endpoint to measure server activity
- Feature: Added configuration options for hub watchdog
  - `--hub-watchdog-provisioned-inactivity-timeout-seconds` Inactivity timeout before a provisioned instance is deprovisioned
  - `--hub-watchdog-hibernated-inactivity-timeout-seconds` Inactivity timeout before a hibernated instance is deprovisioned
  - `--hub-watchdog-inactivity-check-margin-seconds` Margin before timeout at which an activity check is issued
  - `--hub-watchdog-cycle-interval-ms` Watchdog poll interval in milliseconds
  - `--hub-watchdog-cycle-processing-budget-ms` Maximum time budget per watchdog cycle in milliseconds
  - `--hub-watchdog-instance-check-throttle-ms` Minimum delay between checks on a single instance
  - `--hub-watchdog-activity-check-connect-timeout-ms` Connect timeout for activity check requests
  - `--hub-watchdog-activity-check-request-timeout-ms` Request timeout for activity check requests

1 files changed, 77 insertions, 11 deletions

diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
index 269de28c2..314031246 100644
--- a/src/zenserver/hub/zenhubserver.cpp
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -147,6 +147,62 @@ ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options)
 					   cxxopts::value<bool>(m_ServerOptions.HubUseJobObject)->default_value("true"),
 					   "");
 #endif	// ZEN_PLATFORM_WINDOWS
+
+	Options.add_option("hub",
+					   "",
+					   "hub-watchdog-cycle-interval-ms",
+					   "Interval between watchdog cycles in milliseconds",
+					   cxxopts::value<uint32_t>(m_ServerOptions.WatchdogConfig.CycleIntervalMs)->default_value("3000"),
+					   "<ms>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-watchdog-cycle-processing-budget-ms",
+					   "Maximum processing time budget per watchdog cycle in milliseconds",
+					   cxxopts::value<uint32_t>(m_ServerOptions.WatchdogConfig.CycleProcessingBudgetMs)->default_value("500"),
+					   "<ms>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-watchdog-instance-check-throttle-ms",
+					   "Delay between checking successive instances per watchdog cycle in milliseconds",
+					   cxxopts::value<uint32_t>(m_ServerOptions.WatchdogConfig.InstanceCheckThrottleMs)->default_value("5"),
+					   "<ms>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-watchdog-provisioned-inactivity-timeout-seconds",
+					   "Seconds of inactivity after which a provisioned instance is deprovisioned",
+					   cxxopts::value<uint32_t>(m_ServerOptions.WatchdogConfig.ProvisionedInactivityTimeoutSeconds)->default_value("600"),
+					   "<seconds>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-watchdog-hibernated-inactivity-timeout-seconds",
+					   "Seconds of inactivity after which a hibernated instance is deprovisioned",
+					   cxxopts::value<uint32_t>(m_ServerOptions.WatchdogConfig.HibernatedInactivityTimeoutSeconds)->default_value("1800"),
+					   "<seconds>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-watchdog-inactivity-check-margin-seconds",
+					   "Margin in seconds subtracted from inactivity timeout before triggering an activity check",
+					   cxxopts::value<uint32_t>(m_ServerOptions.WatchdogConfig.InactivityCheckMarginSeconds)->default_value("60"),
+					   "<seconds>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-watchdog-activity-check-connect-timeout-ms",
+					   "Connect timeout in milliseconds for instance activity check requests",
+					   cxxopts::value<uint32_t>(m_ServerOptions.WatchdogConfig.ActivityCheckConnectTimeoutMs)->default_value("100"),
+					   "<ms>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-watchdog-activity-check-request-timeout-ms",
+					   "Request timeout in milliseconds for instance activity check requests",
+					   cxxopts::value<uint32_t>(m_ServerOptions.WatchdogConfig.ActivityCheckRequestTimeoutMs)->default_value("200"),
+					   "<ms>");
 }
 
 void
@@ -320,17 +376,27 @@ ZenHubServer::InitializeState(const ZenHubServerConfig& ServerConfig)
 void
 ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig)
 {
-	ZEN_UNUSED(ServerConfig);
-
 	ZEN_INFO("instantiating Hub");
 	m_Hub = std::make_unique<Hub>(
-		Hub::Configuration{.UseJobObject				 = ServerConfig.HubUseJobObject,
-						   .BasePortNumber				 = ServerConfig.HubBasePortNumber,
-						   .InstanceLimit				 = ServerConfig.HubInstanceLimit,
-						   .InstanceHttpThreadCount		 = ServerConfig.HubInstanceHttpThreadCount,
-						   .InstanceCoreLimit			 = ServerConfig.HubInstanceCoreLimit,
-						   .InstanceConfigPath			 = ServerConfig.HubInstanceConfigPath,
-						   .HydrationTargetSpecification = ServerConfig.HydrationTargetSpecification},
+		Hub::Configuration{
+			.UseJobObject				  = ServerConfig.HubUseJobObject,
+			.BasePortNumber				  = ServerConfig.HubBasePortNumber,
+			.InstanceLimit				  = ServerConfig.HubInstanceLimit,
+			.InstanceHttpThreadCount	  = ServerConfig.HubInstanceHttpThreadCount,
+			.InstanceCoreLimit			  = ServerConfig.HubInstanceCoreLimit,
+			.InstanceConfigPath			  = ServerConfig.HubInstanceConfigPath,
+			.HydrationTargetSpecification = ServerConfig.HydrationTargetSpecification,
+			.WatchDog =
+				{
+					.CycleInterval				  = std::chrono::milliseconds(ServerConfig.WatchdogConfig.CycleIntervalMs),
+					.CycleProcessingBudget		  = std::chrono::milliseconds(ServerConfig.WatchdogConfig.CycleProcessingBudgetMs),
+					.InstanceCheckThrottle		  = std::chrono::milliseconds(ServerConfig.WatchdogConfig.InstanceCheckThrottleMs),
+					.ProvisionedInactivityTimeout = std::chrono::seconds(ServerConfig.WatchdogConfig.ProvisionedInactivityTimeoutSeconds),
+					.HibernatedInactivityTimeout  = std::chrono::seconds(ServerConfig.WatchdogConfig.HibernatedInactivityTimeoutSeconds),
+					.InactivityCheckMargin		  = std::chrono::seconds(ServerConfig.WatchdogConfig.InactivityCheckMarginSeconds),
+					.ActivityCheckConnectTimeout  = std::chrono::milliseconds(ServerConfig.WatchdogConfig.ActivityCheckConnectTimeoutMs),
+					.ActivityCheckRequestTimeout  = std::chrono::milliseconds(ServerConfig.WatchdogConfig.ActivityCheckRequestTimeoutMs),
+				}},
 		ZenServerEnvironment(ZenServerEnvironment::Hub,
 							 ServerConfig.DataDir / "hub",
 							 ServerConfig.DataDir / "servers",
@@ -349,10 +415,10 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig)
 	m_ApiService = std::make_unique<zen::HttpApiService>(*m_Http);
 
 	ZEN_INFO("instantiating hub service");
-	m_HubService = std::make_unique<HttpHubService>(*m_Hub);
+	m_HubService = std::make_unique<HttpHubService>(*m_Hub, m_StatsService, m_StatusService);
 	m_HubService->SetNotificationEndpoint(ServerConfig.UpstreamNotificationEndpoint, ServerConfig.InstanceId);
 
-	m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
+	m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatsService, m_StatusService);
 }
 
 void