idle deprovision in hub (#895)

- Feature: Hub watchdog automatically deprovisions inactive provisioned and hibernated instances
- Feature: Added `stats/activity_counters` endpoint to measure server activity
- Feature: Added configuration options for hub watchdog
  - `--hub-watchdog-provisioned-inactivity-timeout-seconds` Inactivity timeout before a provisioned instance is deprovisioned
  - `--hub-watchdog-hibernated-inactivity-timeout-seconds` Inactivity timeout before a hibernated instance is deprovisioned
  - `--hub-watchdog-inactivity-check-margin-seconds` Margin before timeout at which an activity check is issued
  - `--hub-watchdog-cycle-interval-ms` Watchdog poll interval in milliseconds
  - `--hub-watchdog-cycle-processing-budget-ms` Maximum time budget per watchdog cycle in milliseconds
  - `--hub-watchdog-instance-check-throttle-ms` Minimum delay between checks on a single instance
  - `--hub-watchdog-activity-check-connect-timeout-ms` Connect timeout for activity check requests
  - `--hub-watchdog-activity-check-request-timeout-ms` Request timeout for activity check requests

2 files changed, 36 insertions, 8 deletions

diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 697cc014e..fa7b580e8 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -9,6 +9,7 @@
 #include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zencore/trace.h>
+#include <zenhttp/httpstats.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #if ZEN_PLATFORM_WINDOWS
@@ -28,8 +29,9 @@ static unsigned char gHtmlZipData[] = {
 namespace zen {
 
 ////////////////////////////////////////////////////////////////////////////////
-HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService)
+HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatsService& StatsService, HttpStatusService& StatusService)
 : m_Directory(Directory)
+, m_StatsService(StatsService)
 , m_StatusService(StatusService)
 {
 	ZEN_TRACE_CPU("HttpFrontendService::HttpFrontendService");
@@ -94,12 +96,14 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpSt
 	{
 		ZEN_INFO("front-end is NOT AVAILABLE");
 	}
+	m_StatsService.RegisterHandler("dashboard", *this);
 	m_StatusService.RegisterHandler("dashboard", *this);
 }
 
 HttpFrontendService::~HttpFrontendService()
 {
 	m_StatusService.UnregisterHandler("dashboard", *this);
+	m_StatsService.UnregisterHandler("dashboard", *this);
 }
 
 const char*
@@ -122,6 +126,8 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 {
 	using namespace std::literals;
 
+	metrics::OperationTiming::Scope $(m_HttpRequests);
+
 	ExtendableStringBuilder<256> UriBuilder;
 
 	std::string_view Uri = Request.RelativeUriWithExtension();
@@ -230,4 +236,20 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 	}
 }
 
+void
+HttpFrontendService::HandleStatsRequest(HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+
+	EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+uint64_t
+HttpFrontendService::GetActivityCounter()
+{
+	return m_HttpRequests.Count();
+}
+
 }  // namespace zen
diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h
index 0ae3170ad..541e6213b 100644
--- a/src/zenserver/frontend/frontend.h
+++ b/src/zenserver/frontend/frontend.h
@@ -11,20 +11,26 @@
 
 namespace zen {
 
-class HttpFrontendService final : public zen::HttpService, public IHttpStatusProvider
+class HttpStatsService;
+
+class HttpFrontendService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
 {
 public:
-	HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService);
+	HttpFrontendService(std::filesystem::path Directory, HttpStatsService& StatsService, HttpStatusService& StatusService);
 	virtual ~HttpFrontendService();
 	virtual const char* BaseUri() const override;
-	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleRequest(HttpServerRequest& Request) override;
 	virtual void		HandleStatusRequest(HttpServerRequest& Request) override;
+	virtual void		HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual uint64_t	GetActivityCounter() override;
 
 private:
-	std::unique_ptr<ZipFs> m_ZipFs;
-	std::filesystem::path  m_Directory;
-	std::filesystem::path  m_DocsDirectory;
-	HttpStatusService&	   m_StatusService;
+	std::unique_ptr<ZipFs>	 m_ZipFs;
+	std::filesystem::path	 m_Directory;
+	std::filesystem::path	 m_DocsDirectory;
+	HttpStatsService&		 m_StatsService;
+	HttpStatusService&		 m_StatusService;
+	metrics::OperationTiming m_HttpRequests;
 };
 
 }  // namespace zen