adding more stats (#429)

- Feature: Add detailed stats on requests and data sizes on a per-bucket level, use parameter `cachestorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Add detailed stats on requests and data sizes on cidstore, use parameter `cidstorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Dashboard now accepts parameters in the URL which is passed on to the `/stats/z$` endpoint

16 files changed, 551 insertions, 72 deletions

diff --git a/src/zencore/include/zencore/stats.h b/src/zencore/include/zencore/stats.h
index 1a0817b99..e8e08f96a 100644
--- a/src/zencore/include/zencore/stats.h
+++ b/src/zencore/include/zencore/stats.h
@@ -222,6 +222,39 @@ private:
 	Histogram m_Histogram;
 };
 
+struct MeterSnapshot
+{
+	uint64_t Count;
+	double	 MeanRate;
+	double	 Rate1;
+	double	 Rate5;
+	double	 Rate15;
+};
+
+struct HistogramSnapshot
+{
+	double Count;
+	double Avg;
+	double Min;
+	double Max;
+	double P75;
+	double P95;
+	double P99;
+	double P999;
+};
+
+struct StatsSnapshot
+{
+	MeterSnapshot	  Meter;
+	HistogramSnapshot Histogram;
+};
+
+struct RequestStatsSnapshot
+{
+	StatsSnapshot Requests;
+	StatsSnapshot Bytes;
+};
+
 /** Metrics for network requests
 
 	Aggregates tracking of duration, payload sizes into a single
@@ -261,19 +294,39 @@ public:
 
 	struct Scope
 	{
-		Scope(OperationTiming& Outer);
+		Scope(RequestStats& Outer, int64_t Bytes);
 		~Scope();
 
+		void SetBytes(int64_t Bytes) { m_Bytes = Bytes; }
+		void Stop();
 		void Cancel();
 
 	private:
-		OperationTiming& m_Outer;
-		uint64_t		 m_StartTick;
+		RequestStats& m_Outer;
+		uint64_t	  m_StartTick;
+		int64_t		  m_Bytes;
 	};
 
 	void EmitSnapshot(std::string_view Tag, CbObjectWriter& Cbo);
 
+	RequestStatsSnapshot Snapshot();
+
 private:
+	static StatsSnapshot GetSnapshot(Meter& M, Histogram& H, double ConversionFactor)
+	{
+		SampleSnapshot Snap = H.Snapshot();
+		return StatsSnapshot{
+			.Meter	   = {.Count = M.Count(), .MeanRate = M.MeanRate(), .Rate1 = M.Rate1(), .Rate5 = M.Rate5(), .Rate15 = M.Rate15()},
+			.Histogram = {.Count = H.Count() * ConversionFactor,
+						  .Avg	 = H.Mean() * ConversionFactor,
+						  .Min	 = H.Min() * ConversionFactor,
+						  .Max	 = H.Max() * ConversionFactor,
+						  .P75	 = Snap.Get75Percentile() * ConversionFactor,
+						  .P95	 = Snap.Get95Percentile() * ConversionFactor,
+						  .P99	 = Snap.Get99Percentile() * ConversionFactor,
+						  .P999	 = Snap.Get999Percentile() * ConversionFactor}};
+	}
+
 	Meter	  m_RequestMeter;
 	Meter	  m_BytesMeter;
 	Histogram m_RequestTimeHistogram;
@@ -286,6 +339,11 @@ void EmitSnapshot(std::string_view Tag, Meter& Stat, CbObjectWriter& Cbo);
 
 void EmitSnapshot(const Histogram& Stat, CbObjectWriter& Cbo, double ConversionFactor);
 
+void EmitSnapshot(std::string_view Tag, const MeterSnapshot& Snapshot, CbObjectWriter& Cbo);
+void EmitSnapshot(std::string_view Tag, const HistogramSnapshot& Snapshot, CbObjectWriter& Cbo);
+void EmitSnapshot(std::string_view Tag, const StatsSnapshot& Snapshot, CbObjectWriter& Cbo);
+void EmitSnapshot(std::string_view Tag, const RequestStatsSnapshot& Snapshot, CbObjectWriter& Cbo);
+
 }  // namespace zen::metrics
 
 namespace zen {
diff --git a/src/zencore/stats.cpp b/src/zencore/stats.cpp
index 372bc42f8..087319fa4 100644
--- a/src/zencore/stats.cpp
+++ b/src/zencore/stats.cpp
@@ -461,6 +461,31 @@ RequestStats::Count() const
 	return m_RequestMeter.Count();
 }
 
+RequestStats::Scope::Scope(RequestStats& Outer, int64_t Bytes) : m_Outer(Outer), m_StartTick(GetHifreqTimerValue()), m_Bytes(Bytes)
+{
+}
+
+RequestStats::Scope::~Scope()
+{
+	Stop();
+}
+
+void
+RequestStats::Scope::Stop()
+{
+	if (m_StartTick != 0)
+	{
+		m_Outer.Update(GetHifreqTimerValue() - m_StartTick, m_Bytes);
+		m_StartTick = 0;
+	}
+}
+
+void
+RequestStats::Scope::Cancel()
+{
+	m_StartTick = 0;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 void
@@ -489,6 +514,15 @@ RequestStats::EmitSnapshot(std::string_view Tag, CbObjectWriter& Cbo)
 	Cbo.EndObject();
 }
 
+RequestStatsSnapshot
+RequestStats::Snapshot()
+{
+	const double ToSeconds = GetHifreqTimerToSeconds();
+
+	return RequestStatsSnapshot{.Requests = GetSnapshot(m_RequestMeter, m_RequestTimeHistogram, ToSeconds),
+								.Bytes	  = GetSnapshot(m_BytesMeter, m_BytesHistogram, 1.0)};
+}
+
 void
 EmitSnapshot(std::string_view Tag, OperationTiming& Stat, CbObjectWriter& Cbo)
 {
@@ -540,6 +574,44 @@ EmitSnapshot(std::string_view Tag, Meter& Stat, CbObjectWriter& Cbo)
 	Cbo.EndObject();
 }
 
+void
+EmitSnapshot(const MeterSnapshot& Snapshot, CbObjectWriter& Cbo)
+{
+	Cbo << "count" << Snapshot.Count;
+	Cbo << "rate_mean" << Snapshot.MeanRate;
+	Cbo << "rate_1" << Snapshot.Rate1 << "rate_5" << Snapshot.Rate5 << "rate_15" << Snapshot.Rate15;
+}
+
+void
+EmitSnapshot(const HistogramSnapshot& Snapshot, CbObjectWriter& Cbo)
+{
+	Cbo << "t_count" << Snapshot.Count << "t_avg" << Snapshot.Avg;
+	Cbo << "t_min" << Snapshot.Min << "t_max" << Snapshot.Max;
+	Cbo << "t_p75" << Snapshot.P75 << "t_p95" << Snapshot.P95 << "t_p99" << Snapshot.P999;
+}
+
+void
+EmitSnapshot(std::string_view Tag, const StatsSnapshot& Snapshot, CbObjectWriter& Cbo)
+{
+	Cbo.BeginObject(Tag);
+	EmitSnapshot(Snapshot.Meter, Cbo);
+	EmitSnapshot(Snapshot.Histogram, Cbo);
+	Cbo.EndObject();
+}
+
+void
+EmitSnapshot(std::string_view Tag, const RequestStatsSnapshot& Snapshot, CbObjectWriter& Cbo)
+{
+	if (Snapshot.Requests.Meter.Count == 0)
+	{
+		return;
+	}
+	Cbo.BeginObject(Tag);
+	EmitSnapshot("request", Snapshot.Requests, Cbo);
+	EmitSnapshot("bytes", Snapshot.Bytes, Cbo);
+	Cbo.EndObject();
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 #if ZEN_WITH_TESTS
diff --git a/src/zenhttp/monitoring/httpstats.cpp b/src/zenhttp/monitoring/httpstats.cpp
index 1dad48d68..43260a202 100644
--- a/src/zenhttp/monitoring/httpstats.cpp
+++ b/src/zenhttp/monitoring/httpstats.cpp
@@ -70,6 +70,11 @@ HttpStatsService::HandleRequest(HttpServerRequest& Request)
 				if (Key[0] == '/')
 				{
 					Key.remove_prefix(1);
+					size_t SlashPos = Key.find_first_of("/?");
+					if (SlashPos != std::string::npos)
+					{
+						Key = Key.substr(0, SlashPos);
+					}
 
 					RwLock::SharedLockScope _(m_Lock);
 					if (auto It = m_Providers.find(std::string{Key}); It != end(m_Providers))
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 9e6f86d79..3cee3197b 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -662,10 +662,13 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc,
 bool
 ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue)
 {
+	metrics::RequestStats::Scope StatsScope(m_GetOps, 0);
+
 	RwLock::SharedLockScope _(m_IndexLock);
 	auto					It = m_Index.find(HashKey);
 	if (It == m_Index.end())
 	{
+		m_MissCount++;
 		return false;
 	}
 	size_t				 EntryIndex = It.value();
@@ -710,18 +713,31 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 			}
 		}
 	}
-
-	return (bool)OutValue.Value;
+	if (OutValue.Value)
+	{
+		m_HitCount++;
+		StatsScope.SetBytes(OutValue.Value.GetSize());
+		return true;
+	}
+	else
+	{
+		m_MissCount++;
+		return false;
+	}
 }
 
 void
 ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value)
 {
+	metrics::RequestStats::Scope $(m_PutOps, Value.Value.Size());
+
 	if (Value.Value.Size() >= m_LargeObjectThreshold)
 	{
 		return PutStandaloneCacheValue(HashKey, Value);
 	}
 	PutInlineCacheValue(HashKey, Value);
+
+	m_WriteCount++;
 }
 
 bool
@@ -1568,6 +1584,17 @@ ZenCacheDiskLayer::CacheBucket::UpdateAccessTimes(const std::vector<zen::access_
 	}
 }
 
+ZenCacheDiskLayer::BucketStats
+ZenCacheDiskLayer::CacheBucket::Stats()
+{
+	return ZenCacheDiskLayer::BucketStats{.TotalSize  = TotalSize(),
+										  .HitCount	  = m_HitCount,
+										  .MissCount  = m_MissCount,
+										  .WriteCount = m_WriteCount,
+										  .PutOps	  = m_PutOps.Snapshot(),
+										  .GetOps	  = m_GetOps.Snapshot()};
+}
+
 uint64_t
 ZenCacheDiskLayer::CacheBucket::EntryCount() const
 {
@@ -2116,17 +2143,33 @@ ZenCacheDiskLayer::TotalSize() const
 	return TotalSize;
 }
 
+ZenCacheDiskLayer::DiskStats
+ZenCacheDiskLayer::Stats() const
+{
+	ZenCacheDiskLayer::DiskStats Stats = {};
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		Stats.BucketStats.reserve(m_Buckets.size());
+		for (auto& Kv : m_Buckets)
+		{
+			Stats.BucketStats.emplace_back(NamedBucketStats{.BucketName = Kv.first, .Stats = Kv.second->Stats()});
+		}
+	}
+	return Stats;
+}
+
 ZenCacheDiskLayer::Info
 ZenCacheDiskLayer::GetInfo() const
 {
 	ZenCacheDiskLayer::Info Info = {.Config = {.RootDir = m_RootDir}, .TotalSize = TotalSize()};
-
-	RwLock::SharedLockScope _(m_Lock);
-	Info.BucketNames.reserve(m_Buckets.size());
-	for (auto& Kv : m_Buckets)
 	{
-		Info.BucketNames.push_back(Kv.first);
-		Info.EntryCount += Kv.second->EntryCount();
+		RwLock::SharedLockScope _(m_Lock);
+		Info.BucketNames.reserve(m_Buckets.size());
+		for (auto& Kv : m_Buckets)
+		{
+			Info.BucketNames.push_back(Kv.first);
+			Info.EntryCount += Kv.second->EntryCount();
+		}
 	}
 	return Info;
 }
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index fc4d8cd6f..80c643afa 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -4,6 +4,7 @@
 
 #include "cacheshared.h"
 
+#include <zencore/stats.h>
 #include <zenstore/blockstore.h>
 #include <zenstore/caslog.h>
 
@@ -106,6 +107,27 @@ public:
 		uint64_t				 TotalSize	= 0;
 	};
 
+	struct BucketStats
+	{
+		uint64_t					  TotalSize;
+		uint64_t					  HitCount;
+		uint64_t					  MissCount;
+		uint64_t					  WriteCount;
+		metrics::RequestStatsSnapshot PutOps;
+		metrics::RequestStatsSnapshot GetOps;
+	};
+
+	struct NamedBucketStats
+	{
+		std::string BucketName;
+		BucketStats Stats;
+	};
+
+	struct DiskStats
+	{
+		std::vector<NamedBucketStats> BucketStats;
+	};
+
 	explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir);
 	~ZenCacheDiskLayer();
 
@@ -119,8 +141,9 @@ public:
 	void CollectGarbage(GcContext& GcCtx);
 	void UpdateAccessTimes(const zen::access_tracking::AccessTimes& AccessTimes);
 
-	void	 DiscoverBuckets();
-	uint64_t TotalSize() const;
+	void	  DiscoverBuckets();
+	uint64_t  TotalSize() const;
+	DiskStats Stats() const;
 
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
@@ -150,6 +173,7 @@ private:
 
 		inline uint64_t TotalSize() const { return m_TotalStandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(); }
 		uint64_t		EntryCount() const;
+		BucketStats		Stats();
 
 		CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const;
 		void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
@@ -184,6 +208,12 @@ private:
 
 		using IndexMap = tsl::robin_map<IoHash, size_t, IoHash::Hasher>;
 
+		std::atomic<uint64_t> m_HitCount;
+		std::atomic<uint64_t> m_MissCount;
+		std::atomic<uint64_t> m_WriteCount;
+		metrics::RequestStats m_PutOps;
+		metrics::RequestStats m_GetOps;
+
 		mutable RwLock			   m_IndexLock;
 		std::vector<AccessTime>	   m_AccessTimes;
 		std::vector<BucketPayload> m_Payloads;
diff --git a/src/zenserver/cache/cachememorylayer.cpp b/src/zenserver/cache/cachememorylayer.cpp
index c18db7706..be21e60f1 100644
--- a/src/zenserver/cache/cachememorylayer.cpp
+++ b/src/zenserver/cache/cachememorylayer.cpp
@@ -247,6 +247,8 @@ ZenCacheMemoryLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutV
 {
 	ZEN_TRACE_CPU("Z$::Mem::Bucket::Get");
 
+	metrics::OperationTiming::Scope $(m_GetOps);
+
 	RwLock::SharedLockScope _(m_BucketLock);
 
 	if (auto It = m_CacheMap.find(HashKey); It != m_CacheMap.end())
@@ -270,6 +272,8 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue
 {
 	ZEN_TRACE_CPU("Z$::Mem::Bucket::Put");
 
+	metrics::OperationTiming::Scope $(m_PutOps);
+
 	size_t PayloadSize = Value.Value.GetSize();
 	{
 		GcClock::Tick			   AccessTime = GcClock::TickCount();
diff --git a/src/zenserver/cache/cachememorylayer.h b/src/zenserver/cache/cachememorylayer.h
index dc8f22c6f..0ef0c905f 100644
--- a/src/zenserver/cache/cachememorylayer.h
+++ b/src/zenserver/cache/cachememorylayer.h
@@ -6,6 +6,7 @@
 #include "cacheshared.h"
 
 #include <zencore/iohash.h>
+#include <zencore/stats.h>
 #include <zenstore/gc.h>
 
 #include <inttypes.h>
@@ -81,6 +82,9 @@ private:
 		static_assert(sizeof(BucketPayload) == 32u);
 		static_assert(sizeof(AccessTime) == 4u);
 
+		metrics::OperationTiming m_PutOps;
+		metrics::OperationTiming m_GetOps;
+
 		mutable RwLock					 m_BucketLock;
 		std::vector<AccessTime>			 m_AccessTimes;
 		std::vector<BucketPayload>		 m_Payloads;
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index 4a2fdd96b..59ee5e485 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -605,8 +605,6 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 {
 	ZEN_TRACE_CPU("z$::Http::HandleRequest");
 
-	m_CacheStats.RequestCount++;
-
 	metrics::OperationTiming::Scope $(m_HttpRequests);
 
 	std::string_view Key = Request.RelativeUri();
@@ -1608,6 +1606,8 @@ HttpStructuredCacheService::HandleRpcRequest(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::HandleRpcRequest");
 
+	m_CacheStats.RpcRequests.fetch_add(1);
+
 	CbPackage	 Package;
 	CbObjectView Object;
 	CbObject	 ObjectBuffer;
@@ -1838,6 +1838,8 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords([[maybe_unused]] const Cach
 	CbObjectView Params = BatchObject["Params"sv].AsObjectView();
 	CachePolicy	 DefaultPolicy;
 
+	m_CacheStats.RpcRecordRequests.fetch_add(1);
+
 	std::string_view		   PolicyText = Params["DefaultPolicy"].AsString();
 	std::optional<std::string> Namespace  = GetRpcRequestNamespace(Params);
 	if (!Namespace)
@@ -1845,9 +1847,13 @@ HttpStructuredCacheService::HandleRpcPutCacheRecords([[maybe_unused]] const Cach
 		return CbPackage{};
 	}
 	DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
+
 	std::vector<bool> Results;
-	for (CbFieldView RequestField : Params["Requests"sv])
+
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
+	for (CbFieldView RequestField : RequestsArray)
 	{
+		m_CacheStats.RpcRecordBatchRequests.fetch_add(1);
 		CbObjectView RequestObject = RequestField.AsObjectView();
 		CbObjectView RecordObject  = RequestObject["Record"sv].AsObjectView();
 		CbObjectView KeyView	   = RecordObject["Key"sv].AsObjectView();
@@ -1987,6 +1993,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheRecords"sv);
 
 	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
+	m_CacheStats.RpcRecordRequests.fetch_add(1);
 
 	struct ValueRequestData
 	{
@@ -2021,8 +2028,6 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 
 	std::vector<RecordRequestData> Requests;
 	std::vector<size_t>			   UpstreamIndexes;
-	CbArrayView					   RequestsArray = Params["Requests"sv].AsArrayView();
-	Requests.reserve(RequestsArray.Num());
 
 	auto ParseValues = [](RecordRequestData& Request) {
 		CbArrayView ValuesArray = Request.RecordObject["Values"sv].AsArrayView();
@@ -2041,10 +2046,13 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 		}
 	};
 
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
 	for (CbFieldView RequestField : RequestsArray)
 	{
 		ZEN_TRACE_CPU("Z$::RpcGetCacheRecords::Request");
 
+		m_CacheStats.RpcRecordBatchRequests.fetch_add(1);
+
 		Stopwatch		   Timer;
 		RecordRequestData& Request		 = Requests.emplace_back();
 		CbObjectView	   RequestObject = RequestField.AsObjectView();
@@ -2341,6 +2349,8 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CacheRequestContext& C
 	CbObjectView BatchObject = BatchRequest.GetObject();
 	CbObjectView Params		 = BatchObject["Params"sv].AsObjectView();
 
+	m_CacheStats.RpcValueRequests.fetch_add(1);
+
 	std::string_view		   PolicyText	 = Params["DefaultPolicy"].AsString();
 	CachePolicy				   DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
 	std::optional<std::string> Namespace	 = GetRpcRequestNamespace(Params);
@@ -2348,13 +2358,16 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CacheRequestContext& C
 	{
 		return CbPackage{};
 	}
-	const bool HasUpstream = m_UpstreamCache.IsActive();
+	const bool	HasUpstream	  = m_UpstreamCache.IsActive();
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
 
 	std::vector<bool> Results;
-	for (CbFieldView RequestField : Params["Requests"sv])
+	for (CbFieldView RequestField : RequestsArray)
 	{
 		ZEN_TRACE_CPU("Z$::RpcPutCacheValues::Request");
 
+		m_CacheStats.RpcValueBatchRequests.fetch_add(1);
+
 		Stopwatch Timer;
 
 		CbObjectView RequestObject = RequestField.AsObjectView();
@@ -2458,7 +2471,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(const CacheRequestContext& C
 	ZEN_TRACE_CPU("Z$::RpcGetCacheValues");
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheValues"sv);
 
-	CbObjectView			   Params		 = RpcRequest["Params"sv].AsObjectView();
+	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
+	m_CacheStats.RpcValueRequests.fetch_add(1);
+
 	std::string_view		   PolicyText	 = Params["DefaultPolicy"sv].AsString();
 	CachePolicy				   DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default;
 	std::optional<std::string> Namespace	 = GetRpcRequestNamespace(Params);
@@ -2479,14 +2494,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(const CacheRequestContext& C
 
 	std::vector<size_t> RemoteRequestIndexes;
 
-	const bool	HasUpstream = m_UpstreamCache.IsActive();
-	CbArrayView RequestView = Params["Requests"sv].AsArrayView();
-	Requests.reserve(RequestView.Num());
+	const bool HasUpstream = m_UpstreamCache.IsActive();
 
-	for (CbFieldView RequestField : RequestView)
+	CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
+	for (CbFieldView RequestField : RequestsArray)
 	{
 		ZEN_TRACE_CPU("Z$::RpcGetCacheValues::Request");
 
+		m_CacheStats.RpcValueBatchRequests.fetch_add(1);
+
 		Stopwatch Timer;
 
 		RequestData& Request	   = Requests.emplace_back();
@@ -2745,7 +2761,9 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string&								  Nam
 
 	ZEN_ASSERT(RpcRequest["Method"sv].AsString() == "GetCacheChunks"sv);
 
-	CbObjectView	 Params			   = RpcRequest["Params"sv].AsObjectView();
+	CbObjectView Params = RpcRequest["Params"sv].AsObjectView();
+	m_CacheStats.RpcChunkRequests.fetch_add(1);
+
 	std::string_view DefaultPolicyText = Params["DefaultPolicy"sv].AsString();
 	CachePolicy		 DefaultPolicy	   = !DefaultPolicyText.empty() ? ParseCachePolicy(DefaultPolicyText) : CachePolicy::Default;
 
@@ -2759,7 +2777,6 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string&								  Nam
 
 	CbArrayView ChunkRequestsArray = Params["ChunkRequests"sv].AsArrayView();
 	size_t		NumRequests		   = static_cast<size_t>(ChunkRequestsArray.Num());
-
 	// Note that these reservations allow us to take pointers to the elements while populating them. If the reservation is removed,
 	// we will need to change the pointers to indexes to handle reallocations.
 	RecordKeys.reserve(NumRequests);
@@ -2776,6 +2793,8 @@ HttpStructuredCacheService::ParseGetCacheChunksRequest(std::string&								  Nam
 	{
 		ZEN_TRACE_CPU("Z$::ParseGetCacheChunksRequest::Request");
 
+		m_CacheStats.RpcChunkBatchRequests.fetch_add(1);
+
 		CbObjectView	   RequestObject = RequestView.AsObjectView();
 		CacheChunkRequest& RequestKey	 = RequestKeys.emplace_back();
 		ChunkRequest&	   Request		 = Requests.emplace_back();
@@ -3199,13 +3218,11 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	CbObjectWriter Cbo;
 
 	EmitSnapshot("requests", m_HttpRequests, Cbo);
-	EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
 
 	const uint64_t		 HitCount		  = m_CacheStats.HitCount;
 	const uint64_t		 UpstreamHitCount = m_CacheStats.UpstreamHitCount;
 	const uint64_t		 MissCount		  = m_CacheStats.MissCount;
 	const uint64_t		 WriteCount		  = m_CacheStats.WriteCount;
-	const uint64_t		 RequestCount	  = m_CacheStats.RequestCount;
 	const uint64_t		 BadRequestCount  = m_CacheStats.BadRequestCount;
 	struct CidStoreStats StoreStats		  = m_CidStore.Stats();
 	const uint64_t		 ChunkHitCount	  = StoreStats.HitCount;
@@ -3213,11 +3230,52 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	const uint64_t		 ChunkWriteCount  = StoreStats.WriteCount;
 	const uint64_t		 TotalCount		  = HitCount + MissCount;
 
+	const uint64_t RpcRequests			  = m_CacheStats.RpcRequests;
+	const uint64_t RpcRecordRequests	  = m_CacheStats.RpcRecordRequests;
+	const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
+	const uint64_t RpcValueRequests		  = m_CacheStats.RpcValueRequests;
+	const uint64_t RpcValueBatchRequests  = m_CacheStats.RpcValueBatchRequests;
+	const uint64_t RpcChunkRequests		  = m_CacheStats.RpcChunkRequests;
+	const uint64_t RpcChunkBatchRequests  = m_CacheStats.RpcChunkBatchRequests;
+
 	const CidStoreSize	CidSize	  = m_CidStore.TotalSize();
 	const GcStorageSize CacheSize = m_CacheStore.StorageSize();
 
+	bool ShowCidStoreStats	 = Request.GetQueryParams().GetValue("cidstorestats") == "true";
+	bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true";
+
+	CidStoreStats CidStoreStats = {};
+	if (ShowCidStoreStats)
+	{
+		CidStoreStats = m_CidStore.Stats();
+	}
+	ZenCacheStore::CacheStoreStats CacheStoreStats = {};
+	if (ShowCacheStoreStats)
+	{
+		CacheStoreStats = m_CacheStore.Stats();
+	}
+
+	Cbo << "badrequestcount" << BadRequestCount;
+
 	Cbo.BeginObject("cache");
 	{
+		Cbo.BeginObject("rpc");
+		Cbo << "count" << RpcRequests;
+		Cbo << "ops" << RpcRecordRequests + RpcValueRequests + RpcChunkRequests;
+		Cbo.BeginObject("records");
+		Cbo << "count" << RpcRecordRequests;
+		Cbo << "ops" << RpcRecordBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("values");
+		Cbo << "count" << RpcValueRequests;
+		Cbo << "ops" << RpcValueBatchRequests;
+		Cbo.EndObject();
+		Cbo.BeginObject("chunks");
+		Cbo << "chunk" << RpcChunkRequests;
+		Cbo << "ops" << RpcChunkBatchRequests;
+		Cbo.EndObject();
+		Cbo.EndObject();
+
 		Cbo.BeginObject("size");
 		{
 			Cbo << "disk" << CacheSize.DiskSize;
@@ -3225,24 +3283,88 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 		}
 		Cbo.EndObject();
 
-		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
 		Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
 		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
-		Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
-		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-		Cbo << "requestcount" << RequestCount;
-		Cbo << "badrequestcount" << BadRequestCount;
+
+		if (m_UpstreamCache.IsActive())
+		{
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+			Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+			Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+		}
 
 		Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
+
+		if (ShowCacheStoreStats)
+		{
+			Cbo.BeginObject("store");
+			Cbo << "hits" << CacheStoreStats.HitCount << "misses" << CacheStoreStats.MissCount << "writes" << CacheStoreStats.WriteCount;
+			const uint64_t StoreTotal = CacheStoreStats.HitCount + CacheStoreStats.MissCount;
+			Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(CacheStoreStats.HitCount) / double(StoreTotal)) : 0.0);
+			EmitSnapshot("read", CacheStoreStats.GetOps, Cbo);
+			EmitSnapshot("write", CacheStoreStats.PutOps, Cbo);
+			if (!CacheStoreStats.NamespaceStats.empty())
+			{
+				Cbo.BeginArray("namespaces");
+				for (const ZenCacheStore::NamedNamespaceStats& NamespaceStats : CacheStoreStats.NamespaceStats)
+				{
+					Cbo.BeginObject();
+					Cbo.AddString("namespace", NamespaceStats.NamespaceName);
+					Cbo << "hits" << NamespaceStats.Stats.HitCount << "misses" << NamespaceStats.Stats.MissCount << "writes"
+						<< NamespaceStats.Stats.WriteCount;
+					const uint64_t NamespaceTotal = NamespaceStats.Stats.HitCount + NamespaceStats.Stats.MissCount;
+					Cbo << "hit_ratio" << (NamespaceTotal > 0 ? (double(NamespaceStats.Stats.HitCount) / double(NamespaceTotal)) : 0.0);
+					EmitSnapshot("read", NamespaceStats.Stats.GetOps, Cbo);
+					EmitSnapshot("write", NamespaceStats.Stats.PutOps, Cbo);
+					if (!NamespaceStats.Stats.DiskStats.BucketStats.empty())
+					{
+						Cbo.BeginArray("buckets");
+						for (const ZenCacheDiskLayer::NamedBucketStats& BucketStats : NamespaceStats.Stats.DiskStats.BucketStats)
+						{
+							Cbo.BeginObject();
+							Cbo.AddString("bucket", BucketStats.BucketName);
+							if (BucketStats.Stats.TotalSize == 0 && BucketStats.Stats.HitCount == 0 && BucketStats.Stats.MissCount == 0 &&
+								BucketStats.Stats.WriteCount == 0)
+							{
+								Cbo.EndObject();
+								continue;
+							}
+							Cbo << "size" << BucketStats.Stats.TotalSize;
+							const uint64_t BucketTotal = BucketStats.Stats.HitCount + BucketStats.Stats.MissCount;
+							if (BucketTotal == 0 && BucketStats.Stats.WriteCount == 0)
+							{
+								Cbo.EndObject();
+								continue;
+							}
+							Cbo << "hits" << BucketStats.Stats.HitCount << "misses" << BucketStats.Stats.MissCount << "writes"
+								<< BucketStats.Stats.WriteCount;
+							Cbo << "hit_ratio" << (BucketTotal > 0 ? (double(BucketStats.Stats.HitCount) / double(BucketTotal)) : 0.0);
+							EmitSnapshot("read", BucketStats.Stats.GetOps, Cbo);
+							EmitSnapshot("write", BucketStats.Stats.PutOps, Cbo);
+
+							Cbo.EndObject();
+						}
+						Cbo.EndArray();
+					}
+					Cbo.EndObject();
+				}
+				Cbo.EndArray();
+			}
+			Cbo.EndObject();
+		}
+		Cbo.EndObject();
 	}
-	Cbo.EndObject();
 
-	Cbo.BeginObject("upstream");
+	if (m_UpstreamCache.IsActive())
 	{
-		m_UpstreamCache.GetStatus(Cbo);
+		EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
+		Cbo.BeginObject("upstream");
+		{
+			m_UpstreamCache.GetStatus(Cbo);
+		}
+		Cbo.EndObject();
 	}
-	Cbo.EndObject();
 
 	Cbo.BeginObject("cid");
 	{
@@ -3254,6 +3376,16 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 			Cbo << "total" << CidSize.TotalSize;
 		}
 		Cbo.EndObject();
+
+		if (ShowCidStoreStats)
+		{
+			Cbo.BeginObject("store");
+			Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount;
+			EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo);
+			EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo);
+			//		EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo);
+			Cbo.EndObject();
+		}
 	}
 	Cbo.EndObject();
 
diff --git a/src/zenserver/cache/httpstructuredcache.h b/src/zenserver/cache/httpstructuredcache.h
index c64855e70..75ae1711f 100644
--- a/src/zenserver/cache/httpstructuredcache.h
+++ b/src/zenserver/cache/httpstructuredcache.h
@@ -99,8 +99,14 @@ private:
 		std::atomic_uint64_t UpstreamHitCount{};
 		std::atomic_uint64_t MissCount{};
 		std::atomic_uint64_t WriteCount{};
-		std::atomic_uint64_t RequestCount{};
 		std::atomic_uint64_t BadRequestCount{};
+		std::atomic_uint64_t RpcRequests{};
+		std::atomic_uint64_t RpcRecordRequests{};
+		std::atomic_uint64_t RpcRecordBatchRequests{};
+		std::atomic_uint64_t RpcValueRequests{};
+		std::atomic_uint64_t RpcValueBatchRequests{};
+		std::atomic_uint64_t RpcChunkRequests{};
+		std::atomic_uint64_t RpcChunkBatchRequests{};
 	};
 	enum class PutResult
 	{
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 809df1a94..9e14892e3 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -63,12 +63,15 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 {
 	ZEN_TRACE_CPU("Z$::Namespace::Get");
 
+	metrics::RequestStats::Scope StatsScope(m_GetOps, 0);
+
 	bool Ok = m_MemLayer.Get(InBucket, HashKey, OutValue);
 
 	if (Ok)
 	{
 		ZEN_ASSERT(OutValue.Value.Size());
-
+		StatsScope.SetBytes(OutValue.Value.Size());
+		m_HitCount++;
 		return true;
 	}
 
@@ -77,14 +80,18 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 	if (Ok)
 	{
 		ZEN_ASSERT(OutValue.Value.Size());
+		StatsScope.SetBytes(OutValue.Value.Size());
 
 		if (OutValue.Value.Size() <= m_DiskLayerSizeThreshold)
 		{
 			m_MemLayer.Put(InBucket, HashKey, OutValue);
 		}
+		m_HitCount++;
+		return true;
 	}
 
-	return Ok;
+	m_MissCount++;
+	return false;
 }
 
 void
@@ -92,6 +99,8 @@ ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 {
 	ZEN_TRACE_CPU("Z$::Namespace::Put");
 
+	metrics::RequestStats::Scope $(m_PutOps, Value.Value.Size());
+
 	// Store value and index
 
 	ZEN_ASSERT(Value.Value.Size());
@@ -102,6 +111,7 @@ ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 	{
 		m_MemLayer.Put(InBucket, HashKey, Value);
 	}
+	m_WriteCount++;
 }
 
 bool
@@ -219,6 +229,17 @@ ZenCacheNamespace::GetBucketInfo(std::string_view Bucket) const
 	return Info;
 }
 
+ZenCacheNamespace::NamespaceStats
+ZenCacheNamespace::Stats()
+{
+	return ZenCacheNamespace::NamespaceStats{.HitCount	 = m_HitCount,
+											 .MissCount	 = m_MissCount,
+											 .WriteCount = m_WriteCount,
+											 .PutOps	 = m_PutOps.Snapshot(),
+											 .GetOps	 = m_GetOps.Snapshot(),
+											 .DiskStats	 = m_DiskLayer.Stats()};
+}
+
 CacheValueDetails::NamespaceDetails
 ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
 {
@@ -369,6 +390,8 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::Get");
 
+	metrics::RequestStats::Scope OpScope(m_GetOps, 0);
+
 	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
 	{
 		bool Result = Store->Get(Bucket, HashKey, OutValue);
@@ -392,8 +415,15 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 				m_LogEvent.Set();
 			}
 		}
+		if (Result)
+		{
+			m_HitCount++;
+			OpScope.SetBytes(OutValue.Value.GetSize());
+			return true;
+		}
 
-		return Result;
+		m_MissCount++;
+		return false;
 	}
 	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get [{}], bucket '{}', key '{}'",
 			 Context,
@@ -401,6 +431,7 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 			 Bucket,
 			 HashKey.ToHexString());
 
+	m_MissCount++;
 	return false;
 }
 
@@ -413,6 +444,8 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::Put");
 
+	metrics::RequestStats::Scope $(m_PutOps, Value.Value.GetSize());
+
 	if (m_WriteLogEnabled)
 	{
 		ZEN_TRACE_CPU("Z$::Get::WriteLog");
@@ -435,7 +468,9 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 
 	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
 	{
-		return Store->Put(Bucket, HashKey, Value);
+		Store->Put(Bucket, HashKey, Value);
+		m_WriteCount++;
+		return;
 	}
 	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put [{}] bucket '{}', key '{}'",
 			 Context,
@@ -608,6 +643,20 @@ ZenCacheStore::StorageSize() const
 	return Size;
 }
 
+ZenCacheStore::CacheStoreStats
+ZenCacheStore::Stats()
+{
+	ZenCacheStore::CacheStoreStats Result{.HitCount	  = m_HitCount,
+										  .MissCount  = m_MissCount,
+										  .WriteCount = m_WriteCount,
+										  .PutOps	  = m_PutOps.Snapshot(),
+										  .GetOps	  = m_GetOps.Snapshot()};
+	IterateNamespaces([&](std::string_view NamespaceName, ZenCacheNamespace& Store) {
+		Result.NamespaceStats.emplace_back(NamedNamespaceStats{.NamespaceName = std::string(NamespaceName), .Stats = Store.Stats()});
+	});
+	return Result;
+}
+
 void
 ZenCacheStore::SetLoggingConfig(const Configuration::LogConfig& Loggingconfig)
 {
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index e7b64babe..0dd160a98 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -7,6 +7,7 @@
 
 #include <zencore/compactbinary.h>
 #include <zencore/iohash.h>
+#include <zencore/stats.h>
 #include <zenstore/gc.h>
 #include <zenutil/cache/cache.h>
 
@@ -66,6 +67,16 @@ public:
 		ZenCacheMemoryLayer::Info MemoryLayerInfo;
 	};
 
+	struct NamespaceStats
+	{
+		uint64_t					  HitCount;
+		uint64_t					  MissCount;
+		uint64_t					  WriteCount;
+		metrics::RequestStatsSnapshot PutOps;
+		metrics::RequestStatsSnapshot GetOps;
+		ZenCacheDiskLayer::DiskStats  DiskStats;
+	};
+
 	ZenCacheNamespace(GcManager& Gc, const std::filesystem::path& RootDir);
 	~ZenCacheNamespace();
 
@@ -91,6 +102,7 @@ public:
 
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
+	NamespaceStats			  Stats();
 
 	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
 
@@ -98,6 +110,11 @@ private:
 	std::filesystem::path m_RootDir;
 	ZenCacheMemoryLayer	  m_MemLayer;
 	ZenCacheDiskLayer	  m_DiskLayer;
+	std::atomic<uint64_t> m_HitCount{};
+	std::atomic<uint64_t> m_MissCount{};
+	std::atomic<uint64_t> m_WriteCount{};
+	metrics::RequestStats m_PutOps;
+	metrics::RequestStats m_GetOps;
 	uint64_t			  m_DiskLayerSizeThreshold = 1 * 1024;
 	uint64_t			  m_LastScrubTime		   = 0;
 
@@ -138,6 +155,22 @@ public:
 		GcStorageSize			 StorageSize;
 	};
 
+	struct NamedNamespaceStats
+	{
+		std::string						  NamespaceName;
+		ZenCacheNamespace::NamespaceStats Stats;
+	};
+
+	struct CacheStoreStats
+	{
+		uint64_t						 HitCount;
+		uint64_t						 MissCount;
+		uint64_t						 WriteCount;
+		metrics::RequestStatsSnapshot	 PutOps;
+		metrics::RequestStatsSnapshot	 GetOps;
+		std::vector<NamedNamespaceStats> NamespaceStats;
+	};
+
 	ZenCacheStore(GcManager& Gc, const Configuration& Configuration, const DiskWriteBlocker* InDiskWriteBlocker);
 	~ZenCacheStore();
 
@@ -160,7 +193,8 @@ public:
 									  const std::string_view BucketFilter,
 									  const std::string_view ValueFilter) const;
 
-	GcStorageSize StorageSize() const;
+	GcStorageSize	StorageSize() const;
+	CacheStoreStats Stats();
 
 	Configuration								 GetConfiguration() const { return m_Configuration; }
 	void										 SetLoggingConfig(const Configuration::LogConfig& Loggingconfig);
@@ -187,8 +221,13 @@ private:
 	NamespaceMap									m_Namespaces;
 	std::vector<std::unique_ptr<ZenCacheNamespace>> m_DroppedNamespaces;
 
-	GcManager&	  m_Gc;
-	Configuration m_Configuration;
+	GcManager&			  m_Gc;
+	Configuration		  m_Configuration;
+	std::atomic<uint64_t> m_HitCount{};
+	std::atomic<uint64_t> m_MissCount{};
+	std::atomic<uint64_t> m_WriteCount{};
+	metrics::RequestStats m_PutOps;
+	metrics::RequestStats m_GetOps;
 
 	struct AccessLogItem
 	{
diff --git a/src/zenserver/frontend/html/index.html b/src/zenserver/frontend/html/index.html
index 252ee621e..96b69a643 100644
--- a/src/zenserver/frontend/html/index.html
+++ b/src/zenserver/frontend/html/index.html
@@ -12,7 +12,8 @@
 	<script type="text/javascript">
 		const getCacheStats = () => {
 			const opts = { headers: { "Accept": "application/json" } };
-			fetch("/stats/z$", opts)
+			const queryString = window.location.search;
+			fetch("/stats/z$" + queryString, opts)
 				.then(response => {
 					if (!response.ok) {
 						throw Error(response.statusText);
@@ -40,12 +41,12 @@
 		<div class="row">
 			<div class="text-center mt-5">
 				<pre>
-__________                  _________  __
-\____    /  ____    ____   /   _____/_/  |_   ____  _______   ____
-  /     / _/ __ \  /    \  \_____  \ \   __\ /  _ \ \_  __ \_/ __ \
- /     /_ \  ___/ |   |  \ /        \ |  |  (  <_> ) |  | \/\  ___/
+__________                  _________  __                           
+\____    /  ____    ____   /   _____/_/  |_   ____  _______   ____  
+  /     / _/ __ \  /    \  \_____  \ \   __\ /  _ \ \_  __ \_/ __ \ 
+ /     /_ \  ___/ |   |  \ /        \ |  |  (  <_> ) |  | \/\  ___/ 
 /_______ \ \___  >|___|  //_______  / |__|   \____/  |__|    \___  >
-        \/     \/      \/         \/                             \/
+        \/     \/      \/         \/                             \/ 
 				</pre>
 				<pre id="status"/>
 			</div>
@@ -53,7 +54,7 @@ __________                  _________  __
 		<div class="row">
 			<pre class="mb-0">Z$:</pre>
 			<pre id="stats"></pre>
-		<div>
+		</div>
 	</div>
 </body>
 </html>
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 7ab847c73..3edcf5903 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -376,6 +376,9 @@ void
 HttpProjectService::HandleRequest(HttpServerRequest& Request)
 {
 	m_ProjectStats.RequestCount++;
+
+	metrics::OperationTiming::Scope $(m_HttpRequests);
+
 	if (m_Router.HandleRequest(Request) == false)
 	{
 		m_ProjectStats.BadRequestCount++;
@@ -392,6 +395,9 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
 	const CidStoreSize	CidSize	  = m_CidStore.TotalSize();
 
 	CbObjectWriter Cbo;
+
+	EmitSnapshot("requests", m_HttpRequests, Cbo);
+
 	Cbo.BeginObject("store");
 	{
 		Cbo.BeginObject("size");
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 739dabe23..ac12d4d7c 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zencore/stats.h>
 #include <zenhttp/auth/authmgr.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
@@ -82,13 +83,14 @@ private:
 
 	inline spdlog::logger& Log() { return m_Log; }
 
-	spdlog::logger&	  m_Log;
-	CidStore&		  m_CidStore;
-	HttpRequestRouter m_Router;
-	Ref<ProjectStore> m_ProjectStore;
-	HttpStatsService& m_StatsService;
-	AuthMgr&		  m_AuthMgr;
-	ProjectStats	  m_ProjectStats;
+	spdlog::logger&			 m_Log;
+	CidStore&				 m_CidStore;
+	HttpRequestRouter		 m_Router;
+	Ref<ProjectStore>		 m_ProjectStore;
+	HttpStatsService&		 m_StatsService;
+	AuthMgr&				 m_AuthMgr;
+	ProjectStats			 m_ProjectStats;
+	metrics::OperationTiming m_HttpRequests;
 };
 
 }  // namespace zen
diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp
index e366a6cb4..f6560fcff 100644
--- a/src/zenstore/cidstore.cpp
+++ b/src/zenstore/cidstore.cpp
@@ -6,6 +6,7 @@
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
+#include <zencore/stats.h>
 #include <zencore/string.h>
 #include <zenstore/scrubcontext.h>
 
@@ -25,6 +26,8 @@ struct CidStore::Impl
 
 	CidStore::InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, CidStore::InsertMode Mode)
 	{
+		metrics::RequestStats::Scope $(m_AddChunkOps, ChunkData.GetSize());
+
 #if ZEN_BUILD_DEBUG
 		IoHash	 VerifyRawHash;
 		uint64_t _;
@@ -37,26 +40,33 @@ struct CidStore::Impl
 		CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, RawHash, static_cast<CasStore::InsertMode>(Mode));
 		if (Result.New)
 		{
-			WriteCount++;
+			m_WriteCount++;
 		}
 		return {.New = Result.New};
 	}
 
 	IoBuffer FindChunkByCid(const IoHash& DecompressedId)
 	{
+		metrics::RequestStats::Scope StatsScope(m_FindChunkOps, 0);
+
 		IoBuffer Result = m_CasStore.FindChunk(DecompressedId);
 		if (Result)
 		{
-			HitCount++;
+			m_HitCount++;
+			StatsScope.SetBytes(Result.GetSize());
 		}
 		else
 		{
-			MissCount++;
+			m_MissCount++;
 		}
 		return Result;
 	}
 
-	bool ContainsChunk(const IoHash& DecompressedId) { return m_CasStore.ContainsChunk(DecompressedId); }
+	bool ContainsChunk(const IoHash& DecompressedId)
+	{
+		//		metrics::RequestStats::Scope $(m_ContainChunkOps);
+		return m_CasStore.ContainsChunk(DecompressedId);
+	}
 
 	void FilterChunks(HashKeySet& InOutChunks)
 	{
@@ -77,11 +87,25 @@ struct CidStore::Impl
 		m_CasStore.ScrubStorage(Ctx);
 	}
 
-	CidStoreStats Stats() { return CidStoreStats{.HitCount = HitCount, .MissCount = MissCount, .WriteCount = WriteCount}; }
+	CidStoreStats Stats()
+	{
+		return CidStoreStats{
+			.HitCount	  = m_HitCount,
+			.MissCount	  = m_MissCount,
+			.WriteCount	  = m_WriteCount,
+			.AddChunkOps  = m_AddChunkOps.Snapshot(),
+			.FindChunkOps = m_FindChunkOps.Snapshot()
+			//			.ContainChunkOps = m_ContainChunkOps.Snapshot()
+		};
+	}
+
+	std::atomic_uint64_t m_HitCount{};
+	std::atomic_uint64_t m_MissCount{};
+	std::atomic_uint64_t m_WriteCount{};
 
-	std::atomic_uint64_t HitCount{};
-	std::atomic_uint64_t MissCount{};
-	std::atomic_uint64_t WriteCount{};
+	metrics::RequestStats m_AddChunkOps;
+	metrics::RequestStats m_FindChunkOps;
+	//	metrics::OperationTiming m_ContainChunkOps;
 
 	uint64_t m_LastScrubTime = 0;
 };
diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h
index 945f5d25a..1c8d79158 100644
--- a/src/zenstore/include/zenstore/cidstore.h
+++ b/src/zenstore/include/zenstore/cidstore.h
@@ -5,6 +5,7 @@
 #include "zenstore.h"
 
 #include <zencore/iohash.h>
+#include <zencore/stats.h>
 #include <zenstore/hashkeyset.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -31,9 +32,12 @@ struct CidStoreSize
 
 struct CidStoreStats
 {
-	uint64_t HitCount;
-	uint64_t MissCount;
-	uint64_t WriteCount;
+	uint64_t					  HitCount;
+	uint64_t					  MissCount;
+	uint64_t					  WriteCount;
+	metrics::RequestStatsSnapshot AddChunkOps;
+	metrics::RequestStatsSnapshot FindChunkOps;
+	//	metrics::RequestStatsSnapshot ContainChunkOps;
 };
 
 struct CidStoreConfiguration