adding more stats (#429)

- Feature: Add detailed stats on requests and data sizes on a per-bucket level, use parameter `cachestorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Add detailed stats on requests and data sizes on cidstore, use parameter `cidstorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Dashboard now accepts parameters in the URL which is passed on to the `/stats/z$` endpoint

1 files changed, 53 insertions, 4 deletions

diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 809df1a94..9e14892e3 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -63,12 +63,15 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 {
 	ZEN_TRACE_CPU("Z$::Namespace::Get");
 
+	metrics::RequestStats::Scope StatsScope(m_GetOps, 0);
+
 	bool Ok = m_MemLayer.Get(InBucket, HashKey, OutValue);
 
 	if (Ok)
 	{
 		ZEN_ASSERT(OutValue.Value.Size());
-
+		StatsScope.SetBytes(OutValue.Value.Size());
+		m_HitCount++;
 		return true;
 	}
 
@@ -77,14 +80,18 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 	if (Ok)
 	{
 		ZEN_ASSERT(OutValue.Value.Size());
+		StatsScope.SetBytes(OutValue.Value.Size());
 
 		if (OutValue.Value.Size() <= m_DiskLayerSizeThreshold)
 		{
 			m_MemLayer.Put(InBucket, HashKey, OutValue);
 		}
+		m_HitCount++;
+		return true;
 	}
 
-	return Ok;
+	m_MissCount++;
+	return false;
 }
 
 void
@@ -92,6 +99,8 @@ ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 {
 	ZEN_TRACE_CPU("Z$::Namespace::Put");
 
+	metrics::RequestStats::Scope $(m_PutOps, Value.Value.Size());
+
 	// Store value and index
 
 	ZEN_ASSERT(Value.Value.Size());
@@ -102,6 +111,7 @@ ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 	{
 		m_MemLayer.Put(InBucket, HashKey, Value);
 	}
+	m_WriteCount++;
 }
 
 bool
@@ -219,6 +229,17 @@ ZenCacheNamespace::GetBucketInfo(std::string_view Bucket) const
 	return Info;
 }
 
+ZenCacheNamespace::NamespaceStats
+ZenCacheNamespace::Stats()
+{
+	return ZenCacheNamespace::NamespaceStats{.HitCount	 = m_HitCount,
+											 .MissCount	 = m_MissCount,
+											 .WriteCount = m_WriteCount,
+											 .PutOps	 = m_PutOps.Snapshot(),
+											 .GetOps	 = m_GetOps.Snapshot(),
+											 .DiskStats	 = m_DiskLayer.Stats()};
+}
+
 CacheValueDetails::NamespaceDetails
 ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
 {
@@ -369,6 +390,8 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::Get");
 
+	metrics::RequestStats::Scope OpScope(m_GetOps, 0);
+
 	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
 	{
 		bool Result = Store->Get(Bucket, HashKey, OutValue);
@@ -392,8 +415,15 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 				m_LogEvent.Set();
 			}
 		}
+		if (Result)
+		{
+			m_HitCount++;
+			OpScope.SetBytes(OutValue.Value.GetSize());
+			return true;
+		}
 
-		return Result;
+		m_MissCount++;
+		return false;
 	}
 	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get [{}], bucket '{}', key '{}'",
 			 Context,
@@ -401,6 +431,7 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 			 Bucket,
 			 HashKey.ToHexString());
 
+	m_MissCount++;
 	return false;
 }
 
@@ -413,6 +444,8 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::Put");
 
+	metrics::RequestStats::Scope $(m_PutOps, Value.Value.GetSize());
+
 	if (m_WriteLogEnabled)
 	{
 		ZEN_TRACE_CPU("Z$::Get::WriteLog");
@@ -435,7 +468,9 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 
 	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
 	{
-		return Store->Put(Bucket, HashKey, Value);
+		Store->Put(Bucket, HashKey, Value);
+		m_WriteCount++;
+		return;
 	}
 	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put [{}] bucket '{}', key '{}'",
 			 Context,
@@ -608,6 +643,20 @@ ZenCacheStore::StorageSize() const
 	return Size;
 }
 
+ZenCacheStore::CacheStoreStats
+ZenCacheStore::Stats()
+{
+	ZenCacheStore::CacheStoreStats Result{.HitCount	  = m_HitCount,
+										  .MissCount  = m_MissCount,
+										  .WriteCount = m_WriteCount,
+										  .PutOps	  = m_PutOps.Snapshot(),
+										  .GetOps	  = m_GetOps.Snapshot()};
+	IterateNamespaces([&](std::string_view NamespaceName, ZenCacheNamespace& Store) {
+		Result.NamespaceStats.emplace_back(NamedNamespaceStats{.NamespaceName = std::string(NamespaceName), .Stats = Store.Stats()});
+	});
+	return Result;
+}
+
 void
 ZenCacheStore::SetLoggingConfig(const Configuration::LogConfig& Loggingconfig)
 {