adding more stats (#429)

- Feature: Add detailed stats on requests and data sizes on a per-bucket level, use parameter `cachestorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Add detailed stats on requests and data sizes on cidstore, use parameter `cidstorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Dashboard now accepts parameters in the URL which is passed on to the `/stats/z$` endpoint

8 files changed, 351 insertions, 44 deletions

diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 9e6f86d79..3cee3197b 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -662,10 +662,13 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc,
 bool
 ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue)
 {
+	metrics::RequestStats::Scope StatsScope(m_GetOps, 0);
+
 	RwLock::SharedLockScope _(m_IndexLock);
 	auto					It = m_Index.find(HashKey);
 	if (It == m_Index.end())
 	{
+		m_MissCount++;
 		return false;
 	}
 	size_t				 EntryIndex = It.value();
@@ -710,18 +713,31 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 			}
 		}
 	}
-
-	return (bool)OutValue.Value;
+	if (OutValue.Value)
+	{
+		m_HitCount++;
+		StatsScope.SetBytes(OutValue.Value.GetSize());
+		return true;
+	}
+	else
+	{
+		m_MissCount++;
+		return false;
+	}
 }
 
 void
 ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value)
 {
+	metrics::RequestStats::Scope $(m_PutOps, Value.Value.Size());
+
 	if (Value.Value.Size() >= m_LargeObjectThreshold)
 	{
 		return PutStandaloneCacheValue(HashKey, Value);
 	}
 	PutInlineCacheValue(HashKey, Value);
+
+	m_WriteCount++;
 }
 
 bool
@@ -1568,6 +1584,17 @@ ZenCacheDiskLayer::CacheBucket::UpdateAccessTimes(const std::vector<zen::access_
 	}
 }
 
+ZenCacheDiskLayer::BucketStats
+ZenCacheDiskLayer::CacheBucket::Stats()
+{
+	return ZenCacheDiskLayer::BucketStats{.TotalSize  = TotalSize(),
+										  .HitCount	  = m_HitCount,
+										  .MissCount  = m_MissCount,
+										  .WriteCount = m_WriteCount,
+										  .PutOps	  = m_PutOps.Snapshot(),
+										  .GetOps	  = m_GetOps.Snapshot()};
+}
+
 uint64_t
 ZenCacheDiskLayer::CacheBucket::EntryCount() const
 {
@@ -2116,17 +2143,33 @@ ZenCacheDiskLayer::TotalSize() const
 	return TotalSize;
 }
 
+ZenCacheDiskLayer::DiskStats
+ZenCacheDiskLayer::Stats() const
+{
+	ZenCacheDiskLayer::DiskStats Stats = {};
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		Stats.BucketStats.reserve(m_Buckets.size());
+		for (auto& Kv : m_Buckets)
+		{
+			Stats.BucketStats.emplace_back(NamedBucketStats{.BucketName = Kv.first, .Stats = Kv.second->Stats()});
+		}
+	}
+	return Stats;
+}
+
 ZenCacheDiskLayer::Info
 ZenCacheDiskLayer::GetInfo() const
 {
 	ZenCacheDiskLayer::Info Info = {.Config = {.RootDir = m_RootDir}, .TotalSize = TotalSize()};
-
-	RwLock::SharedLockScope _(m_Lock);
-	Info.BucketNames.reserve(m_Buckets.size());
-	for (auto& Kv : m_Buckets)
 	{
-		Info.BucketNames.push_back(Kv.first);
-		Info.EntryCount += Kv.second->EntryCount();
+		RwLock::SharedLockScope _(m_Lock);
+		Info.BucketNames.reserve(m_Buckets.size());
+		for (auto& Kv : m_Buckets)
+		{
+			Info.BucketNames.push_back(Kv.first);
+			Info.EntryCount += Kv.second->EntryCount();
+		}
 	}
 	return Info;
 }
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index fc4d8cd6f..80c643afa 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -4,6 +4,7 @@
 
 #include "cacheshared.h"
 
+#include <zencore/stats.h>
 #include <zenstore/blockstore.h>
 #include <zenstore/caslog.h>
 
@@ -106,6 +107,27 @@ public:
 		uint64_t				 TotalSize	= 0;
 	};
 
+	struct BucketStats
+	{
+		uint64_t					  TotalSize;
+		uint64_t					  HitCount;
+		uint64_t					  MissCount;
+		uint64_t					  WriteCount;
+		metrics::RequestStatsSnapshot PutOps;
+		metrics::RequestStatsSnapshot GetOps;
+	};
+
+	struct NamedBucketStats
+	{
+		std::string BucketName;
+		BucketStats Stats;
+	};
+
+	struct DiskStats
+	{
+		std::vector<NamedBucketStats> BucketStats;
+	};
+
 	explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir);
 	~ZenCacheDiskLayer();
 
@@ -119,8 +141,9 @@ public:
 	void CollectGarbage(GcContext& GcCtx);
 	void UpdateAccessTimes(const zen::access_tracking::AccessTimes& AccessTimes);
 
-	void	 DiscoverBuckets();
-	uint64_t TotalSize() const;
+	void	  DiscoverBuckets();
+	uint64_t  TotalSize() const;
+	DiskStats Stats() const;
 
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
@@ -150,6 +173,7 @@ private:
 
 		inline uint64_t TotalSize() const { return m_TotalStandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(); }
 		uint64_t		EntryCount() const;
+		BucketStats		Stats();
 
 		CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const;
 		void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
@@ -184,6 +208,12 @@ private:
 
 		using IndexMap = tsl::robin_map<IoHash, size_t, IoHash::Hasher>;
 
+		std::atomic<uint64_t> m_HitCount;
+		std::atomic<uint64_t> m_MissCount;
+		std::atomic<uint64_t> m_WriteCount;
+		metrics::RequestStats m_PutOps;
+		metrics::RequestStats m_GetOps;
+
 		mutable RwLock			   m_IndexLock;
 		std::vector<AccessTime>	   m_AccessTimes;
 		std::vector<BucketPayload> m_Payloads;
diff --git a/src/zenserver/cache/cachememorylayer.cpp b/src/zenserver/cache/cachememorylayer.cpp
index c18db7706..be21e60f1 100644
--- a/src/zenserver/cache/cachememorylayer.cpp
+++ b/src/zenserver/cache/cachememorylayer.cpp
@@ -247,6 +247,8 @@ ZenCacheMemoryLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutV
 {
 	ZEN_TRACE_CPU("Z$::Mem::Bucket::Get");
 
+	metrics::OperationTiming::Scope $(m_GetOps);
+
 	RwLock::SharedLockScope _(m_BucketLock);
 
 	if (auto It = m_CacheMap.find(HashKey); It != m_CacheMap.end())
@@ -270,6 +272,8 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue
 {
 	ZEN_TRACE_CPU("Z$::Mem::Bucket::Put");
 
+	metrics::OperationTiming::Scope $(m_PutOps);
+
 	size_t PayloadSize = Value.Value.GetSize();
 	{
 		GcClock::Tick			   AccessTime = GcClock::TickCount();
diff --git a/src/zenserver/cache/cachememorylayer.h b/src/zenserver/cache/cachememorylayer.h
index dc8f22c6f..0ef0c905f 100644
--- a/src/zenserver/cache/cachememorylayer.h
+++ b/src/zenserver/cache/cachememorylayer.h
@@ -6,6 +6,7 @@
 #include "cacheshared.h"
 
 #include <zencore/iohash.h>
+#include <zencore/stats.h>
 #include <zenstore/gc.h>
 
 #include <inttypes.h>
@@ -81,6 +82,9 @@ private:
 		static_assert(sizeof(BucketPayload) == 32u);
 		static_assert(sizeof(AccessTime) == 4u);
 
+		metrics::OperationTiming m_PutOps;
+		metrics::OperationTiming m_GetOps;
+
 		mutable RwLock					 m_BucketLock;
 		std::vector<AccessTime>			 m_AccessTimes;
 		std::vector<BucketPayload>		 m_Payloads;
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index 4a2fdd96b..59ee5e485 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -605,8 +605,6 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 {
 	ZEN_TRACE_CPU("z$::Http::HandleRequest");
 
-	m_CacheStats.RequestCount++;
-
 	metrics::OperationTiming::Scope $(m_HttpRequests);
 
 	std::string_view Key = Request.RelativeUri();
@@ -1608,6 +1606,8 @@ HttpStructuredCacheService::HandleRpcRequest(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::HandleRpcRequest");
 
+	m_CacheStats.RpcRequests.fetch_add(1);
+
 	CbPackage	 Package;
 	CbObjectView Object;
 	CbObject	 ObjectBuffer;
@@ -1838,6 +1838,8 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords([[maybe_unused]] const Cach
 	CbObjectView Params = BatchObject["Params"sv].AsObjectView();
 	CachePolicy	 DefaultPolicy;
 
+	m_CacheStats.RpcRecordRequests.fetch_add(1);
+
 	std::string_view		   PolicyText = Params["DefaultPolicy"].AsString();
 	std::optional<std::string> Namespace  = GetRpcRequestNamespace(Params);
 	if (!Namespace)
@@ -1845,9 +1847,13 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords([[maybe_unused]] const Cach
 		return CbPackage{};
 	}
 	DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
+
 	std::vector<bool> Results;
-	for (CbFieldView RequestField : Params["Requests"sv])
+
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
+	for (CbFieldView RequestField : RequestsArray)
 	{
+		m_CacheStats.RpcRecordBatchRequests.fetch_add(1);
 		CbObjectView RequestObject = RequestField.AsObjectView();
 		CbObjectView RecordObject  = RequestObject["Record"sv].AsObjectView();
 		CbObjectView KeyView	   = RecordObject["Key"sv].AsObjectView();
@@ -1987,6 +1993,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheRecords"sv);
 
 	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
+	m_CacheStats.RpcRecordRequests.fetch_add(1);
 
 	struct ValueRequestData
 	{
@@ -2021,8 +2028,6 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 
 	std::vector<RecordRequestData> Requests;
 	std::vector<size_t>			   UpstreamIndexes;
-	CbArrayView					   RequestsArray = Params["Requests"sv].AsArrayView();
-	Requests.reserve(RequestsArray.Num());
 
 	auto ParseValues = [](RecordRequestData& Request) {
 		CbArrayView ValuesArray = Request.RecordObject["Values"sv].AsArrayView();
@@ -2041,10 +2046,13 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 		}
 	};
 
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
 	for (CbFieldView RequestField : RequestsArray)
 	{
 		ZEN_TRACE_CPU("Z$::RpcGetCacheRecords::Request");
 
+		m_CacheStats.RpcRecordBatchRequests.fetch_add(1);
+
 		Stopwatch		   Timer;
 		RecordRequestData& Request		 = Requests.emplace_back();
 		CbObjectView	   RequestObject = RequestField.AsObjectView();
@@ -2341,6 +2349,8 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CacheRequestContext& C
 	CbObjectView BatchObject = BatchRequest.GetObject();
 	CbObjectView Params		 = BatchObject["Params"sv].AsObjectView();
 
+	m_CacheStats.RpcValueRequests.fetch_add(1);
+
 	std::string_view		   PolicyText	 = Params["DefaultPolicy"].AsString();
 	CachePolicy				   DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
 	std::optional<std::string> Namespace	 = GetRpcRequestNamespace(Params);
@@ -2348,13 +2358,16 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CacheRequestContext& C
 	{
 		return CbPackage{};
 	}
-	const bool HasUpstream = m_UpstreamCache.IsActive();
+	const bool	HasUpstream	  = m_UpstreamCache.IsActive();
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
 
 	std::vector<bool> Results;
-	for (CbFieldView RequestField : Params["Requests"sv])
+	for (CbFieldView RequestField : RequestsArray)
 	{
 		ZEN_TRACE_CPU("Z$::RpcPutCacheValues::Request");
 
+		m_CacheStats.RpcValueBatchRequests.fetch_add(1);
+
 		Stopwatch Timer;
 
 		CbObjectView RequestObject = RequestField.AsObjectView();
@@ -2458,7 +2471,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(const CacheRequestContext& C
 	ZEN_TRACE_CPU("Z$::RpcGetCacheValues");
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheValues"sv);
 
-	CbObjectView			   Params		 = RpcRequest["Params"sv].AsObjectView();
+	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
+	m_CacheStats.RpcValueRequests.fetch_add(1);
+
 	std::string_view		   PolicyText	 = Params["DefaultPolicy"sv].AsString();
 	CachePolicy				   DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
 	std::optional<std::string> Namespace	 = GetRpcRequestNamespace(Params);
@@ -2479,14 +2494,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(const CacheRequestContext& C
 
 	std::vector<size_t> RemoteRequestIndexes;
 
-	const bool	HasUpstream = m_UpstreamCache.IsActive();
-	CbArrayView RequestView = Params["Requests"sv].AsArrayView();
-	Requests.reserve(RequestView.Num());
+	const bool HasUpstream = m_UpstreamCache.IsActive();
 
-	for (CbFieldView RequestField : RequestView)
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
+	for (CbFieldView RequestField : RequestsArray)
 	{
 		ZEN_TRACE_CPU("Z$::RpcGetCacheValues::Request");
 
+		m_CacheStats.RpcValueBatchRequests.fetch_add(1);
+
 		Stopwatch Timer;
 
 		RequestData& Request	   = Requests.emplace_back();
@@ -2745,7 +2761,9 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string&								  Nam
 
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheChunks"sv);
 
-	CbObjectView	 Params			   = RpcRequest["Params"sv].AsObjectView();
+	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
+	m_CacheStats.RpcChunkRequests.fetch_add(1);
+
 	std::string_view DefaultPolicyText = Params["DefaultPolicy"sv].AsString();
 	CachePolicy		 DefaultPolicy	   = !DefaultPolicyText.empty() ? ParseCachePolicy(DefaultPolicyText) : CachePolicy::Default;
 
@@ -2759,7 +2777,6 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string&								  Nam
 
 	CbArrayView ChunkRequestsArray = Params["ChunkRequests"sv].AsArrayView();
 	size_t		NumRequests		   = static_cast<size_t>(ChunkRequestsArray.Num());
-
 	// Note that these reservations allow us to take pointers to the elements while populating them. If the reservation is removed,
 	// we will need to change the pointers to indexes to handle reallocations.
 	RecordKeys.reserve(NumRequests);
@@ -2776,6 +2793,8 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string&								  Nam
 	{
 		ZEN_TRACE_CPU("Z$::ParseGetCacheChunksRequest::Request");
 
+		m_CacheStats.RpcChunkBatchRequests.fetch_add(1);
+
 		CbObjectView	   RequestObject = RequestView.AsObjectView();
 		CacheChunkRequest& RequestKey	 = RequestKeys.emplace_back();
 		ChunkRequest&	   Request		 = Requests.emplace_back();
@@ -3199,13 +3218,11 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	CbObjectWriter Cbo;
 
 	EmitSnapshot("requests", m_HttpRequests, Cbo);
-	EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
 
 	const uint64_t		 HitCount		  = m_CacheStats.HitCount;
 	const uint64_t		 UpstreamHitCount = m_CacheStats.UpstreamHitCount;
 	const uint64_t		 MissCount		  = m_CacheStats.MissCount;
 	const uint64_t		 WriteCount		  = m_CacheStats.WriteCount;
-	const uint64_t		 RequestCount	  = m_CacheStats.RequestCount;
 	const uint64_t		 BadRequestCount  = m_CacheStats.BadRequestCount;
 	struct CidStoreStats StoreStats		  = m_CidStore.Stats();
 	const uint64_t		 ChunkHitCount	  = StoreStats.HitCount;
@@ -3213,11 +3230,52 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	const uint64_t		 ChunkWriteCount  = StoreStats.WriteCount;
 	const uint64_t		 TotalCount		  = HitCount + MissCount;
 
+	const uint64_t RpcRequests			  = m_CacheStats.RpcRequests;
+	const uint64_t RpcRecordRequests	  = m_CacheStats.RpcRecordRequests;
+	const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
+	const uint64_t RpcValueRequests		  = m_CacheStats.RpcValueRequests;
+	const uint64_t RpcValueBatchRequests  = m_CacheStats.RpcValueBatchRequests;
+	const uint64_t RpcChunkRequests		  = m_CacheStats.RpcChunkRequests;
+	const uint64_t RpcChunkBatchRequests  = m_CacheStats.RpcChunkBatchRequests;
+
 	const CidStoreSize	CidSize	  = m_CidStore.TotalSize();
 	const GcStorageSize CacheSize = m_CacheStore.StorageSize();
 
+	bool ShowCidStoreStats	 = Request.GetQueryParams().GetValue("cidstorestats") == "true";
+	bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true";
+
+	CidStoreStats CidStoreStats = {};
+	if (ShowCidStoreStats)
+	{
+		CidStoreStats = m_CidStore.Stats();
+	}
+	ZenCacheStore::CacheStoreStats CacheStoreStats = {};
+	if (ShowCacheStoreStats)
+	{
+		CacheStoreStats = m_CacheStore.Stats();
+	}
+
+	Cbo << "badrequestcount" << BadRequestCount;
+
 	Cbo.BeginObject("cache");
 	{
+		Cbo.BeginObject("rpc");
+		Cbo << "count" << RpcRequests;
+		Cbo << "ops" << RpcRecordRequests + RpcValueRequests + RpcChunkRequests;
+		Cbo.BeginObject("records");
+		Cbo << "count" << RpcRecordRequests;
+		Cbo << "ops" << RpcRecordBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("values");
+		Cbo << "count" << RpcValueRequests;
+		Cbo << "ops" << RpcValueBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("chunks");
+		Cbo << "chunk" << RpcChunkRequests;
+		Cbo << "ops" << RpcChunkBatchRequests;
+		Cbo.EndObject();
+		Cbo.EndObject();
+
 		Cbo.BeginObject("size");
 		{
 			Cbo << "disk" << CacheSize.DiskSize;
@@ -3225,24 +3283,88 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 		}
 		Cbo.EndObject();
 
-		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
 		Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
 		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
-		Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
-		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-		Cbo << "requestcount" << RequestCount;
-		Cbo << "badrequestcount" << BadRequestCount;
+
+		if (m_UpstreamCache.IsActive())
+		{
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+			Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+		}
 
 		Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
+
+		if (ShowCacheStoreStats)
+		{
+			Cbo.BeginObject("store");
+			Cbo << "hits" << CacheStoreStats.HitCount << "misses" << CacheStoreStats.MissCount << "writes" << CacheStoreStats.WriteCount;
+			const uint64_t StoreTotal = CacheStoreStats.HitCount + CacheStoreStats.MissCount;
+			Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(CacheStoreStats.HitCount) / double(StoreTotal)) : 0.0);
+			EmitSnapshot("read", CacheStoreStats.GetOps, Cbo);
+			EmitSnapshot("write", CacheStoreStats.PutOps, Cbo);
+			if (!CacheStoreStats.NamespaceStats.empty())
+			{
+				Cbo.BeginArray("namespaces");
+				for (const ZenCacheStore::NamedNamespaceStats& NamespaceStats : CacheStoreStats.NamespaceStats)
+				{
+					Cbo.BeginObject();
+					Cbo.AddString("namespace", NamespaceStats.NamespaceName);
+					Cbo << "hits" << NamespaceStats.Stats.HitCount << "misses" << NamespaceStats.Stats.MissCount << "writes"
+						<< NamespaceStats.Stats.WriteCount;
+					const uint64_t NamespaceTotal = NamespaceStats.Stats.HitCount + NamespaceStats.Stats.MissCount;
+					Cbo << "hit_ratio" << (NamespaceTotal > 0 ? (double(NamespaceStats.Stats.HitCount) / double(NamespaceTotal)) : 0.0);
+					EmitSnapshot("read", NamespaceStats.Stats.GetOps, Cbo);
+					EmitSnapshot("write", NamespaceStats.Stats.PutOps, Cbo);
+					if (!NamespaceStats.Stats.DiskStats.BucketStats.empty())
+					{
+						Cbo.BeginArray("buckets");
+						for (const ZenCacheDiskLayer::NamedBucketStats& BucketStats : NamespaceStats.Stats.DiskStats.BucketStats)
+						{
+							Cbo.BeginObject();
+							Cbo.AddString("bucket", BucketStats.BucketName);
+							if (BucketStats.Stats.TotalSize == 0 && BucketStats.Stats.HitCount == 0 && BucketStats.Stats.MissCount == 0 &&
+								BucketStats.Stats.WriteCount == 0)
+							{
+								Cbo.EndObject();
+								continue;
+							}
+							Cbo << "size" << BucketStats.Stats.TotalSize;
+							const uint64_t BucketTotal = BucketStats.Stats.HitCount + BucketStats.Stats.MissCount;
+							if (BucketTotal == 0 && BucketStats.Stats.WriteCount == 0)
+							{
+								Cbo.EndObject();
+								continue;
+							}
+							Cbo << "hits" << BucketStats.Stats.HitCount << "misses" << BucketStats.Stats.MissCount << "writes"
+								<< BucketStats.Stats.WriteCount;
+							Cbo << "hit_ratio" << (BucketTotal > 0 ? (double(BucketStats.Stats.HitCount) / double(BucketTotal)) : 0.0);
+							EmitSnapshot("read", BucketStats.Stats.GetOps, Cbo);
+							EmitSnapshot("write", BucketStats.Stats.PutOps, Cbo);
+
+							Cbo.EndObject();
+						}
+						Cbo.EndArray();
+					}
+					Cbo.EndObject();
+				}
+				Cbo.EndArray();
+			}
+			Cbo.EndObject();
+		}
+		Cbo.EndObject();
 	}
-	Cbo.EndObject();
 
-	Cbo.BeginObject("upstream");
+	if (m_UpstreamCache.IsActive())
 	{
-		m_UpstreamCache.GetStatus(Cbo);
+		EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
+		Cbo.BeginObject("upstream");
+		{
+			m_UpstreamCache.GetStatus(Cbo);
+		}
+		Cbo.EndObject();
 	}
-	Cbo.EndObject();
 
 	Cbo.BeginObject("cid");
 	{
@@ -3254,6 +3376,16 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 			Cbo << "total" << CidSize.TotalSize;
 		}
 		Cbo.EndObject();
+
+		if (ShowCidStoreStats)
+		{
+			Cbo.BeginObject("store");
+			Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount;
+			EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo);
+			EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo);
+			//		EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo);
+			Cbo.EndObject();
+		}
 	}
 	Cbo.EndObject();
 
diff --git a/src/zenserver/cache/httpstructuredcache.h b/src/zenserver/cache/httpstructuredcache.h
index c64855e70..75ae1711f 100644
--- a/src/zenserver/cache/httpstructuredcache.h
+++ b/src/zenserver/cache/httpstructuredcache.h
@@ -99,8 +99,14 @@ private:
 		std::atomic_uint64_t UpstreamHitCount{};
 		std::atomic_uint64_t MissCount{};
 		std::atomic_uint64_t WriteCount{};
-		std::atomic_uint64_t RequestCount{};
 		std::atomic_uint64_t BadRequestCount{};
+		std::atomic_uint64_t RpcRequests{};
+		std::atomic_uint64_t RpcRecordRequests{};
+		std::atomic_uint64_t RpcRecordBatchRequests{};
+		std::atomic_uint64_t RpcValueRequests{};
+		std::atomic_uint64_t RpcValueBatchRequests{};
+		std::atomic_uint64_t RpcChunkRequests{};
+		std::atomic_uint64_t RpcChunkBatchRequests{};
 	};
 	enum class PutResult
 	{
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 809df1a94..9e14892e3 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -63,12 +63,15 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 {
 	ZEN_TRACE_CPU("Z$::Namespace::Get");
 
+	metrics::RequestStats::Scope StatsScope(m_GetOps, 0);
+
 	bool Ok = m_MemLayer.Get(InBucket, HashKey, OutValue);
 
 	if (Ok)
 	{
 		ZEN_ASSERT(OutValue.Value.Size());
-
+		StatsScope.SetBytes(OutValue.Value.Size());
+		m_HitCount++;
 		return true;
 	}
 
@@ -77,14 +80,18 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 	if (Ok)
 	{
 		ZEN_ASSERT(OutValue.Value.Size());
+		StatsScope.SetBytes(OutValue.Value.Size());
 
 		if (OutValue.Value.Size() <= m_DiskLayerSizeThreshold)
 		{
 			m_MemLayer.Put(InBucket, HashKey, OutValue);
 		}
+		m_HitCount++;
+		return true;
 	}
 
-	return Ok;
+	m_MissCount++;
+	return false;
 }
 
 void
@@ -92,6 +99,8 @@ ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 {
 	ZEN_TRACE_CPU("Z$::Namespace::Put");
 
+	metrics::RequestStats::Scope $(m_PutOps, Value.Value.Size());
+
 	// Store value and index
 
 	ZEN_ASSERT(Value.Value.Size());
@@ -102,6 +111,7 @@ ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 	{
 		m_MemLayer.Put(InBucket, HashKey, Value);
 	}
+	m_WriteCount++;
 }
 
 bool
@@ -219,6 +229,17 @@ ZenCacheNamespace::GetBucketInfo(std::string_view Bucket) const
 	return Info;
 }
 
+ZenCacheNamespace::NamespaceStats
+ZenCacheNamespace::Stats()
+{
+	return ZenCacheNamespace::NamespaceStats{.HitCount	 = m_HitCount,
+											 .MissCount	 = m_MissCount,
+											 .WriteCount = m_WriteCount,
+											 .PutOps	 = m_PutOps.Snapshot(),
+											 .GetOps	 = m_GetOps.Snapshot(),
+											 .DiskStats	 = m_DiskLayer.Stats()};
+}
+
 CacheValueDetails::NamespaceDetails
 ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
 {
@@ -369,6 +390,8 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::Get");
 
+	metrics::RequestStats::Scope OpScope(m_GetOps, 0);
+
 	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
 	{
 		bool Result = Store->Get(Bucket, HashKey, OutValue);
@@ -392,8 +415,15 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 				m_LogEvent.Set();
 			}
 		}
+		if (Result)
+		{
+			m_HitCount++;
+			OpScope.SetBytes(OutValue.Value.GetSize());
+			return true;
+		}
 
-		return Result;
+		m_MissCount++;
+		return false;
 	}
 	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get [{}], bucket '{}', key '{}'",
 			 Context,
@@ -401,6 +431,7 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 			 Bucket,
 			 HashKey.ToHexString());
 
+	m_MissCount++;
 	return false;
 }
 
@@ -413,6 +444,8 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::Put");
 
+	metrics::RequestStats::Scope $(m_PutOps, Value.Value.GetSize());
+
 	if (m_WriteLogEnabled)
 	{
 		ZEN_TRACE_CPU("Z$::Get::WriteLog");
@@ -435,7 +468,9 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 
 	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
 	{
-		return Store->Put(Bucket, HashKey, Value);
+		Store->Put(Bucket, HashKey, Value);
+		m_WriteCount++;
+		return;
 	}
 	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put [{}] bucket '{}', key '{}'",
 			 Context,
@@ -608,6 +643,20 @@ ZenCacheStore::StorageSize() const
 	return Size;
 }
 
+ZenCacheStore::CacheStoreStats
+ZenCacheStore::Stats()
+{
+	ZenCacheStore::CacheStoreStats Result{.HitCount	  = m_HitCount,
+										  .MissCount  = m_MissCount,
+										  .WriteCount = m_WriteCount,
+										  .PutOps	  = m_PutOps.Snapshot(),
+										  .GetOps	  = m_GetOps.Snapshot()};
+	IterateNamespaces([&](std::string_view NamespaceName, ZenCacheNamespace& Store) {
+		Result.NamespaceStats.emplace_back(NamedNamespaceStats{.NamespaceName = std::string(NamespaceName), .Stats = Store.Stats()});
+	});
+	return Result;
+}
+
 void
 ZenCacheStore::SetLoggingConfig(const Configuration::LogConfig& Loggingconfig)
 {
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index e7b64babe..0dd160a98 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -7,6 +7,7 @@
 
 #include <zencore/compactbinary.h>
 #include <zencore/iohash.h>
+#include <zencore/stats.h>
 #include <zenstore/gc.h>
 #include <zenutil/cache/cache.h>
 
@@ -66,6 +67,16 @@ public:
 		ZenCacheMemoryLayer::Info MemoryLayerInfo;
 	};
 
+	struct NamespaceStats
+	{
+		uint64_t					  HitCount;
+		uint64_t					  MissCount;
+		uint64_t					  WriteCount;
+		metrics::RequestStatsSnapshot PutOps;
+		metrics::RequestStatsSnapshot GetOps;
+		ZenCacheDiskLayer::DiskStats  DiskStats;
+	};
+
 	ZenCacheNamespace(GcManager& Gc, const std::filesystem::path& RootDir);
 	~ZenCacheNamespace();
 
@@ -91,6 +102,7 @@ public:
 
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
+	NamespaceStats			  Stats();
 
 	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
 
@@ -98,6 +110,11 @@ private:
 	std::filesystem::path m_RootDir;
 	ZenCacheMemoryLayer	  m_MemLayer;
 	ZenCacheDiskLayer	  m_DiskLayer;
+	std::atomic<uint64_t> m_HitCount{};
+	std::atomic<uint64_t> m_MissCount{};
+	std::atomic<uint64_t> m_WriteCount{};
+	metrics::RequestStats m_PutOps;
+	metrics::RequestStats m_GetOps;
 	uint64_t			  m_DiskLayerSizeThreshold = 1 * 1024;
 	uint64_t			  m_LastScrubTime		   = 0;
 
@@ -138,6 +155,22 @@ public:
 		GcStorageSize			 StorageSize;
 	};
 
+	struct NamedNamespaceStats
+	{
+		std::string						  NamespaceName;
+		ZenCacheNamespace::NamespaceStats Stats;
+	};
+
+	struct CacheStoreStats
+	{
+		uint64_t						 HitCount;
+		uint64_t						 MissCount;
+		uint64_t						 WriteCount;
+		metrics::RequestStatsSnapshot	 PutOps;
+		metrics::RequestStatsSnapshot	 GetOps;
+		std::vector<NamedNamespaceStats> NamespaceStats;
+	};
+
 	ZenCacheStore(GcManager& Gc, const Configuration& Configuration, const DiskWriteBlocker* InDiskWriteBlocker);
 	~ZenCacheStore();
 
@@ -160,7 +193,8 @@ public:
 									  const std::string_view BucketFilter,
 									  const std::string_view ValueFilter) const;
 
-	GcStorageSize StorageSize() const;
+	GcStorageSize	StorageSize() const;
+	CacheStoreStats Stats();
 
 	Configuration								 GetConfiguration() const { return m_Configuration; }
 	void										 SetLoggingConfig(const Configuration::LogConfig& Loggingconfig);
@@ -187,8 +221,13 @@ private:
 	NamespaceMap									m_Namespaces;
 	std::vector<std::unique_ptr<ZenCacheNamespace>> m_DroppedNamespaces;
 
-	GcManager&	  m_Gc;
-	Configuration m_Configuration;
+	GcManager&			  m_Gc;
+	Configuration		  m_Configuration;
+	std::atomic<uint64_t> m_HitCount{};
+	std::atomic<uint64_t> m_MissCount{};
+	std::atomic<uint64_t> m_WriteCount{};
+	metrics::RequestStats m_PutOps;
+	metrics::RequestStats m_GetOps;
 
 	struct AccessLogItem
 	{