adding more stats (#429)

- Feature: Add detailed stats on requests and data sizes on a per-bucket level, use parameter `cachestorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Add detailed stats on requests and data sizes on cidstore, use parameter `cidstorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Dashboard now accepts parameters in the URL which is passed on to the `/stats/z$` endpoint

1 files changed, 158 insertions, 26 deletions

diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index 4a2fdd96b..59ee5e485 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -605,8 +605,6 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 {
 	ZEN_TRACE_CPU("z$::Http::HandleRequest");
 
-	m_CacheStats.RequestCount++;
-
 	metrics::OperationTiming::Scope $(m_HttpRequests);
 
 	std::string_view Key = Request.RelativeUri();
@@ -1608,6 +1606,8 @@ HttpStructuredCacheService::HandleRpcRequest(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::HandleRpcRequest");
 
+	m_CacheStats.RpcRequests.fetch_add(1);
+
 	CbPackage	 Package;
 	CbObjectView Object;
 	CbObject	 ObjectBuffer;
@@ -1838,6 +1838,8 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords([[maybe_unused]] const Cach
 	CbObjectView Params = BatchObject["Params"sv].AsObjectView();
 	CachePolicy	 DefaultPolicy;
 
+	m_CacheStats.RpcRecordRequests.fetch_add(1);
+
 	std::string_view		   PolicyText = Params["DefaultPolicy"].AsString();
 	std::optional<std::string> Namespace  = GetRpcRequestNamespace(Params);
 	if (!Namespace)
@@ -1845,9 +1847,13 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords([[maybe_unused]] const Cach
 		return CbPackage{};
 	}
 	DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
+
 	std::vector<bool> Results;
-	for (CbFieldView RequestField : Params["Requests"sv])
+
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
+	for (CbFieldView RequestField : RequestsArray)
 	{
+		m_CacheStats.RpcRecordBatchRequests.fetch_add(1);
 		CbObjectView RequestObject = RequestField.AsObjectView();
 		CbObjectView RecordObject  = RequestObject["Record"sv].AsObjectView();
 		CbObjectView KeyView	   = RecordObject["Key"sv].AsObjectView();
@@ -1987,6 +1993,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheRecords"sv);
 
 	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
+	m_CacheStats.RpcRecordRequests.fetch_add(1);
 
 	struct ValueRequestData
 	{
@@ -2021,8 +2028,6 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 
 	std::vector<RecordRequestData> Requests;
 	std::vector<size_t>			   UpstreamIndexes;
-	CbArrayView					   RequestsArray = Params["Requests"sv].AsArrayView();
-	Requests.reserve(RequestsArray.Num());
 
 	auto ParseValues = [](RecordRequestData& Request) {
 		CbArrayView ValuesArray = Request.RecordObject["Values"sv].AsArrayView();
@@ -2041,10 +2046,13 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 		}
 	};
 
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
 	for (CbFieldView RequestField : RequestsArray)
 	{
 		ZEN_TRACE_CPU("Z$::RpcGetCacheRecords::Request");
 
+		m_CacheStats.RpcRecordBatchRequests.fetch_add(1);
+
 		Stopwatch		   Timer;
 		RecordRequestData& Request		 = Requests.emplace_back();
 		CbObjectView	   RequestObject = RequestField.AsObjectView();
@@ -2341,6 +2349,8 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CacheRequestContext& C
 	CbObjectView BatchObject = BatchRequest.GetObject();
 	CbObjectView Params		 = BatchObject["Params"sv].AsObjectView();
 
+	m_CacheStats.RpcValueRequests.fetch_add(1);
+
 	std::string_view		   PolicyText	 = Params["DefaultPolicy"].AsString();
 	CachePolicy				   DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
 	std::optional<std::string> Namespace	 = GetRpcRequestNamespace(Params);
@@ -2348,13 +2358,16 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CacheRequestContext& C
 	{
 		return CbPackage{};
 	}
-	const bool HasUpstream = m_UpstreamCache.IsActive();
+	const bool	HasUpstream	  = m_UpstreamCache.IsActive();
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
 
 	std::vector<bool> Results;
-	for (CbFieldView RequestField : Params["Requests"sv])
+	for (CbFieldView RequestField : RequestsArray)
 	{
 		ZEN_TRACE_CPU("Z$::RpcPutCacheValues::Request");
 
+		m_CacheStats.RpcValueBatchRequests.fetch_add(1);
+
 		Stopwatch Timer;
 
 		CbObjectView RequestObject = RequestField.AsObjectView();
@@ -2458,7 +2471,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(const CacheRequestContext& C
 	ZEN_TRACE_CPU("Z$::RpcGetCacheValues");
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheValues"sv);
 
-	CbObjectView			   Params		 = RpcRequest["Params"sv].AsObjectView();
+	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
+	m_CacheStats.RpcValueRequests.fetch_add(1);
+
 	std::string_view		   PolicyText	 = Params["DefaultPolicy"sv].AsString();
 	CachePolicy				   DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
 	std::optional<std::string> Namespace	 = GetRpcRequestNamespace(Params);
@@ -2479,14 +2494,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(const CacheRequestContext& C
 
 	std::vector<size_t> RemoteRequestIndexes;
 
-	const bool	HasUpstream = m_UpstreamCache.IsActive();
-	CbArrayView RequestView = Params["Requests"sv].AsArrayView();
-	Requests.reserve(RequestView.Num());
+	const bool HasUpstream = m_UpstreamCache.IsActive();
 
-	for (CbFieldView RequestField : RequestView)
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
+	for (CbFieldView RequestField : RequestsArray)
 	{
 		ZEN_TRACE_CPU("Z$::RpcGetCacheValues::Request");
 
+		m_CacheStats.RpcValueBatchRequests.fetch_add(1);
+
 		Stopwatch Timer;
 
 		RequestData& Request	   = Requests.emplace_back();
@@ -2745,7 +2761,9 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string&								  Nam
 
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheChunks"sv);
 
-	CbObjectView	 Params			   = RpcRequest["Params"sv].AsObjectView();
+	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
+	m_CacheStats.RpcChunkRequests.fetch_add(1);
+
 	std::string_view DefaultPolicyText = Params["DefaultPolicy"sv].AsString();
 	CachePolicy		 DefaultPolicy	   = !DefaultPolicyText.empty() ? ParseCachePolicy(DefaultPolicyText) : CachePolicy::Default;
 
@@ -2759,7 +2777,6 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string&								  Nam
 
 	CbArrayView ChunkRequestsArray = Params["ChunkRequests"sv].AsArrayView();
 	size_t		NumRequests		   = static_cast<size_t>(ChunkRequestsArray.Num());
-
 	// Note that these reservations allow us to take pointers to the elements while populating them. If the reservation is removed,
 	// we will need to change the pointers to indexes to handle reallocations.
 	RecordKeys.reserve(NumRequests);
@@ -2776,6 +2793,8 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string&								  Nam
 	{
 		ZEN_TRACE_CPU("Z$::ParseGetCacheChunksRequest::Request");
 
+		m_CacheStats.RpcChunkBatchRequests.fetch_add(1);
+
 		CbObjectView	   RequestObject = RequestView.AsObjectView();
 		CacheChunkRequest& RequestKey	 = RequestKeys.emplace_back();
 		ChunkRequest&	   Request		 = Requests.emplace_back();
@@ -3199,13 +3218,11 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	CbObjectWriter Cbo;
 
 	EmitSnapshot("requests", m_HttpRequests, Cbo);
-	EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
 
 	const uint64_t		 HitCount		  = m_CacheStats.HitCount;
 	const uint64_t		 UpstreamHitCount = m_CacheStats.UpstreamHitCount;
 	const uint64_t		 MissCount		  = m_CacheStats.MissCount;
 	const uint64_t		 WriteCount		  = m_CacheStats.WriteCount;
-	const uint64_t		 RequestCount	  = m_CacheStats.RequestCount;
 	const uint64_t		 BadRequestCount  = m_CacheStats.BadRequestCount;
 	struct CidStoreStats StoreStats		  = m_CidStore.Stats();
 	const uint64_t		 ChunkHitCount	  = StoreStats.HitCount;
@@ -3213,11 +3230,52 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	const uint64_t		 ChunkWriteCount  = StoreStats.WriteCount;
 	const uint64_t		 TotalCount		  = HitCount + MissCount;
 
+	const uint64_t RpcRequests			  = m_CacheStats.RpcRequests;
+	const uint64_t RpcRecordRequests	  = m_CacheStats.RpcRecordRequests;
+	const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
+	const uint64_t RpcValueRequests		  = m_CacheStats.RpcValueRequests;
+	const uint64_t RpcValueBatchRequests  = m_CacheStats.RpcValueBatchRequests;
+	const uint64_t RpcChunkRequests		  = m_CacheStats.RpcChunkRequests;
+	const uint64_t RpcChunkBatchRequests  = m_CacheStats.RpcChunkBatchRequests;
+
 	const CidStoreSize	CidSize	  = m_CidStore.TotalSize();
 	const GcStorageSize CacheSize = m_CacheStore.StorageSize();
 
+	bool ShowCidStoreStats	 = Request.GetQueryParams().GetValue("cidstorestats") == "true";
+	bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true";
+
+	CidStoreStats CidStoreStats = {};
+	if (ShowCidStoreStats)
+	{
+		CidStoreStats = m_CidStore.Stats();
+	}
+	ZenCacheStore::CacheStoreStats CacheStoreStats = {};
+	if (ShowCacheStoreStats)
+	{
+		CacheStoreStats = m_CacheStore.Stats();
+	}
+
+	Cbo << "badrequestcount" << BadRequestCount;
+
 	Cbo.BeginObject("cache");
 	{
+		Cbo.BeginObject("rpc");
+		Cbo << "count" << RpcRequests;
+		Cbo << "ops" << RpcRecordRequests + RpcValueRequests + RpcChunkRequests;
+		Cbo.BeginObject("records");
+		Cbo << "count" << RpcRecordRequests;
+		Cbo << "ops" << RpcRecordBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("values");
+		Cbo << "count" << RpcValueRequests;
+		Cbo << "ops" << RpcValueBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("chunks");
+		Cbo << "chunk" << RpcChunkRequests;
+		Cbo << "ops" << RpcChunkBatchRequests;
+		Cbo.EndObject();
+		Cbo.EndObject();
+
 		Cbo.BeginObject("size");
 		{
 			Cbo << "disk" << CacheSize.DiskSize;
@@ -3225,24 +3283,88 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 		}
 		Cbo.EndObject();
 
-		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
 		Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
 		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
-		Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
-		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-		Cbo << "requestcount" << RequestCount;
-		Cbo << "badrequestcount" << BadRequestCount;
+
+		if (m_UpstreamCache.IsActive())
+		{
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+			Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+		}
 
 		Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
+
+		if (ShowCacheStoreStats)
+		{
+			Cbo.BeginObject("store");
+			Cbo << "hits" << CacheStoreStats.HitCount << "misses" << CacheStoreStats.MissCount << "writes" << CacheStoreStats.WriteCount;
+			const uint64_t StoreTotal = CacheStoreStats.HitCount + CacheStoreStats.MissCount;
+			Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(CacheStoreStats.HitCount) / double(StoreTotal)) : 0.0);
+			EmitSnapshot("read", CacheStoreStats.GetOps, Cbo);
+			EmitSnapshot("write", CacheStoreStats.PutOps, Cbo);
+			if (!CacheStoreStats.NamespaceStats.empty())
+			{
+				Cbo.BeginArray("namespaces");
+				for (const ZenCacheStore::NamedNamespaceStats& NamespaceStats : CacheStoreStats.NamespaceStats)
+				{
+					Cbo.BeginObject();
+					Cbo.AddString("namespace", NamespaceStats.NamespaceName);
+					Cbo << "hits" << NamespaceStats.Stats.HitCount << "misses" << NamespaceStats.Stats.MissCount << "writes"
+						<< NamespaceStats.Stats.WriteCount;
+					const uint64_t NamespaceTotal = NamespaceStats.Stats.HitCount + NamespaceStats.Stats.MissCount;
+					Cbo << "hit_ratio" << (NamespaceTotal > 0 ? (double(NamespaceStats.Stats.HitCount) / double(NamespaceTotal)) : 0.0);
+					EmitSnapshot("read", NamespaceStats.Stats.GetOps, Cbo);
+					EmitSnapshot("write", NamespaceStats.Stats.PutOps, Cbo);
+					if (!NamespaceStats.Stats.DiskStats.BucketStats.empty())
+					{
+						Cbo.BeginArray("buckets");
+						for (const ZenCacheDiskLayer::NamedBucketStats& BucketStats : NamespaceStats.Stats.DiskStats.BucketStats)
+						{
+							Cbo.BeginObject();
+							Cbo.AddString("bucket", BucketStats.BucketName);
+							if (BucketStats.Stats.TotalSize == 0 && BucketStats.Stats.HitCount == 0 && BucketStats.Stats.MissCount == 0 &&
+								BucketStats.Stats.WriteCount == 0)
+							{
+								Cbo.EndObject();
+								continue;
+							}
+							Cbo << "size" << BucketStats.Stats.TotalSize;
+							const uint64_t BucketTotal = BucketStats.Stats.HitCount + BucketStats.Stats.MissCount;
+							if (BucketTotal == 0 && BucketStats.Stats.WriteCount == 0)
+							{
+								Cbo.EndObject();
+								continue;
+							}
+							Cbo << "hits" << BucketStats.Stats.HitCount << "misses" << BucketStats.Stats.MissCount << "writes"
+								<< BucketStats.Stats.WriteCount;
+							Cbo << "hit_ratio" << (BucketTotal > 0 ? (double(BucketStats.Stats.HitCount) / double(BucketTotal)) : 0.0);
+							EmitSnapshot("read", BucketStats.Stats.GetOps, Cbo);
+							EmitSnapshot("write", BucketStats.Stats.PutOps, Cbo);
+
+							Cbo.EndObject();
+						}
+						Cbo.EndArray();
+					}
+					Cbo.EndObject();
+				}
+				Cbo.EndArray();
+			}
+			Cbo.EndObject();
+		}
+		Cbo.EndObject();
 	}
-	Cbo.EndObject();
 
-	Cbo.BeginObject("upstream");
+	if (m_UpstreamCache.IsActive())
 	{
-		m_UpstreamCache.GetStatus(Cbo);
+		EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
+		Cbo.BeginObject("upstream");
+		{
+			m_UpstreamCache.GetStatus(Cbo);
+		}
+		Cbo.EndObject();
 	}
-	Cbo.EndObject();
 
 	Cbo.BeginObject("cid");
 	{
@@ -3254,6 +3376,16 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 			Cbo << "total" << CidSize.TotalSize;
 		}
 		Cbo.EndObject();
+
+		if (ShowCidStoreStats)
+		{
+			Cbo.BeginObject("store");
+			Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount;
+			EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo);
+			EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo);
+			//		EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo);
+			Cbo.EndObject();
+		}
 	}
 	Cbo.EndObject();