idle deprovision in hub (#895)

- Feature: Hub watchdog automatically deprovisions inactive provisioned and hibernated instances
- Feature: Added `stats/activity_counters` endpoint to measure server activity
- Feature: Added configuration options for hub watchdog
  - `--hub-watchdog-provisioned-inactivity-timeout-seconds` Inactivity timeout before a provisioned instance is deprovisioned
  - `--hub-watchdog-hibernated-inactivity-timeout-seconds` Inactivity timeout before a hibernated instance is deprovisioned
  - `--hub-watchdog-inactivity-check-margin-seconds` Margin before timeout at which an activity check is issued
  - `--hub-watchdog-cycle-interval-ms` Watchdog poll interval in milliseconds
  - `--hub-watchdog-cycle-processing-budget-ms` Maximum time budget per watchdog cycle in milliseconds
  - `--hub-watchdog-instance-check-throttle-ms` Minimum delay between checks on a single instance
  - `--hub-watchdog-activity-check-connect-timeout-ms` Connect timeout for activity check requests
  - `--hub-watchdog-activity-check-request-timeout-ms` Request timeout for activity check requests

1 files changed, 23 insertions, 1 deletions

diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 697cc014e..fa7b580e8 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -9,6 +9,7 @@
 #include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zencore/trace.h>
+#include <zenhttp/httpstats.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #if ZEN_PLATFORM_WINDOWS
@@ -28,8 +29,9 @@ static unsigned char gHtmlZipData[] = {
 namespace zen {
 
 ////////////////////////////////////////////////////////////////////////////////
-HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService)
+HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatsService& StatsService, HttpStatusService& StatusService)
 : m_Directory(Directory)
+, m_StatsService(StatsService)
 , m_StatusService(StatusService)
 {
 	ZEN_TRACE_CPU("HttpFrontendService::HttpFrontendService");
@@ -94,12 +96,14 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpSt
 	{
 		ZEN_INFO("front-end is NOT AVAILABLE");
 	}
+	m_StatsService.RegisterHandler("dashboard", *this);
 	m_StatusService.RegisterHandler("dashboard", *this);
 }
 
 HttpFrontendService::~HttpFrontendService()
 {
 	m_StatusService.UnregisterHandler("dashboard", *this);
+	m_StatsService.UnregisterHandler("dashboard", *this);
 }
 
 const char*
@@ -122,6 +126,8 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 {
 	using namespace std::literals;
 
+	metrics::OperationTiming::Scope $(m_HttpRequests);
+
 	ExtendableStringBuilder<256> UriBuilder;
 
 	std::string_view Uri = Request.RelativeUriWithExtension();
@@ -230,4 +236,20 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 	}
 }
 
+void
+HttpFrontendService::HandleStatsRequest(HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+
+	EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+uint64_t
+HttpFrontendService::GetActivityCounter()
+{
+	return m_HttpRequests.Count();
+}
+
 }  // namespace zen