stats: Implemented new stats endpoint

Stats are exposed under /stats/{id}, so for example structured cache stats are exposed under /stats/z$

The separate endpoint makes it easier to separate request handling to ensure stats/status endpoints still respond if the regular request queue is somehow saturated or otherwise not behaving

There is also a /status endpoint which is similar and is targeted towards lightweight health monitoring

8 files changed, 99 insertions, 23 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index d6174caf6..27bb1c5cd 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -12,6 +12,7 @@
 #include <zenhttp/httpserver.h>
 #include <zenstore/CAS.h>
 
+#include "monitoring/httpstats.h"
 #include "structuredcache.h"
 #include "structuredcachestore.h"
 #include "upstream/jupiter.h"
@@ -149,13 +150,19 @@ ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams)
 HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore&				  InCacheStore,
 													   CasStore&					  InStore,
 													   CidStore&					  InCidStore,
+													   HttpStatsService&			  StatsService,
+													   HttpStatusService&			  StatusService,
 													   std::unique_ptr<UpstreamCache> UpstreamCache)
 : m_Log(logging::Get("cache"))
 , m_CacheStore(InCacheStore)
+, m_StatsService(StatsService)
+, m_StatusService(StatusService)
 , m_CasStore(InStore)
 , m_CidStore(InCidStore)
 , m_UpstreamCache(std::move(UpstreamCache))
 {
+	StatsService.RegisterHandler("z$", *this);
+	StatusService.RegisterHandler("z$", *this);
 }
 
 HttpStructuredCacheService::~HttpStructuredCacheService()
@@ -200,13 +207,6 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 	{
 		std::string_view Key = Request.RelativeUri();
 
-		if (Key.empty() || Key == "stats.json")
-		{
-			$.Cancel();
-
-			return HandleStatusRequest(Request);
-		}
-
 		if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); }))
 		{
 			// Bucket reference
@@ -870,10 +870,9 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef&
 }
 
 void
-HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request)
+HttpStructuredCacheService::HandleStatsRequest(zen::HttpServerRequest& Request)
 {
 	CbObjectWriter Cbo;
-	Cbo << "ok" << true;
 
 	EmitSnapshot("requests", m_HttpRequests, Cbo);
 
@@ -899,4 +898,12 @@ HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request)
 	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
+void
+HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
 }  // namespace zen
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index a360878bd..703e24ed3 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -5,6 +5,9 @@
 #include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 
+#include "monitoring/httpstats.h"
+#include "monitoring/httpstatus.h"
+
 #include <memory>
 
 namespace spdlog {
@@ -47,12 +50,14 @@ enum class CachePolicy : uint8_t;
  *
  */
 
-class HttpStructuredCacheService : public zen::HttpService
+class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider
 {
 public:
 	HttpStructuredCacheService(ZenCacheStore&				  InCacheStore,
-							   zen::CasStore&				  InCasStore,
-							   zen::CidStore&				  InCidStore,
+							   CasStore&					  InCasStore,
+							   CidStore&					  InCidStore,
+							   HttpStatsService&			  StatsService,
+							   HttpStatusService&			  StatusService,
 							   std::unique_ptr<UpstreamCache> UpstreamCache);
 	~HttpStructuredCacheService();
 
@@ -86,13 +91,16 @@ private:
 	void			   HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
 	void			   HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
 	void			   HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket);
-	void			   HandleStatusRequest(zen::HttpServerRequest& Request);
+	virtual void	   HandleStatsRequest(zen::HttpServerRequest& Request) override;
+	virtual void	   HandleStatusRequest(zen::HttpServerRequest& Request) override;
 
 	spdlog::logger&				   Log() { return m_Log; }
 	spdlog::logger&				   m_Log;
-	zen::ZenCacheStore&			   m_CacheStore;
-	zen::CasStore&				   m_CasStore;
-	zen::CidStore&				   m_CidStore;
+	ZenCacheStore&				   m_CacheStore;
+	HttpStatsService&			   m_StatsService;
+	HttpStatusService&			   m_StatusService;
+	CasStore&					   m_CasStore;
+	CidStore&					   m_CidStore;
 	std::unique_ptr<UpstreamCache> m_UpstreamCache;
 	uint64_t					   m_LastScrubTime = 0;
 	metrics::OperationTiming	   m_HttpRequests;
diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp
index 79fcf0a17..98d570cfe 100644
--- a/zenserver/experimental/frontend.cpp
+++ b/zenserver/experimental/frontend.cpp
@@ -24,7 +24,7 @@ body {
 <script type="text/javascript">
 	const getCacheStats = () => {
 		const opts = { headers: { "Accept": "application/json" } };
-		fetch("/z$", opts)
+		fetch("/stats/z$", opts)
 			.then(response => {
 				if (!response.ok) {
 					throw Error(response.statusText);
diff --git a/zenserver/monitoring/httpstats.cpp b/zenserver/monitoring/httpstats.cpp
index a82cfda50..de04294d0 100644
--- a/zenserver/monitoring/httpstats.cpp
+++ b/zenserver/monitoring/httpstats.cpp
@@ -19,14 +19,31 @@ HttpStatsService::BaseUri() const
 }
 
 void
+HttpStatsService::RegisterHandler(std::string_view Id, IHttpStatsProvider& Provider)
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+	m_Providers.insert_or_assign(std::string(Id), &Provider);
+}
+
+void
 HttpStatsService::HandleRequest(HttpServerRequest& Request)
 {
 	using namespace std::literals;
 
+	std::string_view Key = Request.RelativeUri();
+
 	switch (Request.RequestVerb())
 	{
+		case HttpVerb::kHead:
 		case HttpVerb::kGet:
-			return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, u8"OK!"sv);
+			if (auto It = m_Providers.find(std::string{Key}); It != end(m_Providers))
+			{
+				return It->second->HandleStatsRequest(Request);
+			}
+
+			[[fallthrough]];
+		default:
+			return;
 	}
 }
 
diff --git a/zenserver/monitoring/httpstats.h b/zenserver/monitoring/httpstats.h
index 9da48233b..1c3c79dd0 100644
--- a/zenserver/monitoring/httpstats.h
+++ b/zenserver/monitoring/httpstats.h
@@ -2,11 +2,18 @@
 
 #pragma once
 
-#include <zenhttp/httpserver.h>
 #include <zencore/logging.h>
+#include <zenhttp/httpserver.h>
+
+#include <map>
 
 namespace zen {
 
+struct IHttpStatsProvider
+{
+	virtual void HandleStatsRequest(HttpServerRequest& Request) = 0;
+};
+
 class HttpStatsService : public HttpService
 {
 public:
@@ -15,12 +22,16 @@ public:
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& Request) override;
+	void				RegisterHandler(std::string_view Id, IHttpStatsProvider& Provider);
 
 private:
 	spdlog::logger&	  m_Log;
 	HttpRequestRouter m_Router;
 
 	inline spdlog::logger& Log() { return m_Log; }
+
+	RwLock									   m_Lock;
+	std::map<std::string, IHttpStatsProvider*> m_Providers;
 };
 
 }  // namespace zen
\ No newline at end of file
diff --git a/zenserver/monitoring/httpstatus.cpp b/zenserver/monitoring/httpstatus.cpp
index c18bf6c1f..e12662b1c 100644
--- a/zenserver/monitoring/httpstatus.cpp
+++ b/zenserver/monitoring/httpstatus.cpp
@@ -15,7 +15,14 @@ HttpStatusService::~HttpStatusService()
 const char*
 HttpStatusService::BaseUri() const
 {
-	return "/statUs/";
+	return "/status/";
+}
+
+void
+HttpStatusService::RegisterHandler(std::string_view Id, IHttpStatusProvider& Provider)
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+	m_Providers.insert_or_assign(std::string(Id), &Provider);
 }
 
 void
@@ -23,10 +30,20 @@ HttpStatusService::HandleRequest(HttpServerRequest& Request)
 {
 	using namespace std::literals;
 
+	std::string_view Key = Request.RelativeUri();
+
 	switch (Request.RequestVerb())
 	{
+		case HttpVerb::kHead:
 		case HttpVerb::kGet:
-			return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, u8"OK!"sv);
+			if (auto It = m_Providers.find(std::string{Key}); It != end(m_Providers))
+			{
+				return It->second->HandleStatusRequest(Request);
+			}
+
+			[[fallthrough]];
+		default:
+			return;
 	}
 }
 
diff --git a/zenserver/monitoring/httpstatus.h b/zenserver/monitoring/httpstatus.h
index f5c6bd616..8f069f760 100644
--- a/zenserver/monitoring/httpstatus.h
+++ b/zenserver/monitoring/httpstatus.h
@@ -2,11 +2,18 @@
 
 #pragma once
 
-#include <zenhttp/httpserver.h>
 #include <zencore/logging.h>
+#include <zenhttp/httpserver.h>
+
+#include <map>
 
 namespace zen {
 
+struct IHttpStatusProvider
+{
+	virtual void HandleStatusRequest(HttpServerRequest& Request) = 0;
+};
+
 class HttpStatusService : public HttpService
 {
 public:
@@ -15,11 +22,15 @@ public:
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& Request) override;
+	void				RegisterHandler(std::string_view Id, IHttpStatusProvider& Provider);
 
 private:
 	spdlog::logger&	  m_Log;
 	HttpRequestRouter m_Router;
 
+	RwLock										m_Lock;
+	std::map<std::string, IHttpStatusProvider*> m_Providers;
+
 	inline spdlog::logger& Log() { return m_Log; }
 };
 
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 0bec56593..f283a2644 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -503,7 +503,12 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
 		}
 	}
 
-	m_StructuredCacheService.reset(new zen::HttpStructuredCacheService(*m_CacheStore, *m_CasStore, *m_CidStore, std::move(UpstreamCache)));
+	m_StructuredCacheService.reset(new zen::HttpStructuredCacheService(*m_CacheStore,
+																	   *m_CasStore,
+																	   *m_CidStore,
+																	   m_StatsService,
+																	   m_StatusService,
+																	   std::move(UpstreamCache)));
 }
 
 }  // namespace zen