From bf20e4c8e63638792e69098d4d9810c1136ff627 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Thu, 28 Sep 2023 23:57:31 +0200
Subject: adding more stats (#429)

- Feature: Add detailed stats on requests and data sizes on a per-bucket level, use parameter `cachestorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Add detailed stats on requests and data sizes on cidstore, use parameter `cidstorestats=true` on the `/stats/z$` endpoint to enable
- Feature: Dashboard now accepts parameters in the URL which is passed on to the `/stats/z$` endpoint
---
 src/zenstore/cidstore.cpp                | 40 +++++++++++++++++++++++++-------
 src/zenstore/include/zenstore/cidstore.h | 10 +++++---
 2 files changed, 39 insertions(+), 11 deletions(-)

(limited to 'src/zenstore')

diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp
index e366a6cb4..f6560fcff 100644
--- a/src/zenstore/cidstore.cpp
+++ b/src/zenstore/cidstore.cpp
@@ -6,6 +6,7 @@
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
+#include <zencore/stats.h>
 #include <zencore/string.h>
 #include <zenstore/scrubcontext.h>
 
@@ -25,6 +26,8 @@ struct CidStore::Impl
 
 	CidStore::InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, CidStore::InsertMode Mode)
 	{
+		metrics::RequestStats::Scope $(m_AddChunkOps, ChunkData.GetSize());
+
 #if ZEN_BUILD_DEBUG
 		IoHash	 VerifyRawHash;
 		uint64_t _;
@@ -37,26 +40,33 @@ struct CidStore::Impl
 		CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, RawHash, static_cast<CasStore::InsertMode>(Mode));
 		if (Result.New)
 		{
-			WriteCount++;
+			m_WriteCount++;
 		}
 		return {.New = Result.New};
 	}
 
 	IoBuffer FindChunkByCid(const IoHash& DecompressedId)
 	{
+		metrics::RequestStats::Scope StatsScope(m_FindChunkOps, 0);
+
 		IoBuffer Result = m_CasStore.FindChunk(DecompressedId);
 		if (Result)
 		{
-			HitCount++;
+			m_HitCount++;
+			StatsScope.SetBytes(Result.GetSize());
 		}
 		else
 		{
-			MissCount++;
+			m_MissCount++;
 		}
 		return Result;
 	}
 
-	bool ContainsChunk(const IoHash& DecompressedId) { return m_CasStore.ContainsChunk(DecompressedId); }
+	bool ContainsChunk(const IoHash& DecompressedId)
+	{
+		//		metrics::RequestStats::Scope $(m_ContainChunkOps);
+		return m_CasStore.ContainsChunk(DecompressedId);
+	}
 
 	void FilterChunks(HashKeySet& InOutChunks)
 	{
@@ -77,11 +87,25 @@ struct CidStore::Impl
 		m_CasStore.ScrubStorage(Ctx);
 	}
 
-	CidStoreStats Stats() { return CidStoreStats{.HitCount = HitCount, .MissCount = MissCount, .WriteCount = WriteCount}; }
+	CidStoreStats Stats()
+	{
+		return CidStoreStats{
+			.HitCount	  = m_HitCount,
+			.MissCount	  = m_MissCount,
+			.WriteCount	  = m_WriteCount,
+			.AddChunkOps  = m_AddChunkOps.Snapshot(),
+			.FindChunkOps = m_FindChunkOps.Snapshot()
+			//			.ContainChunkOps = m_ContainChunkOps.Snapshot()
+		};
+	}
+
+	std::atomic_uint64_t m_HitCount{};
+	std::atomic_uint64_t m_MissCount{};
+	std::atomic_uint64_t m_WriteCount{};
 
-	std::atomic_uint64_t HitCount{};
-	std::atomic_uint64_t MissCount{};
-	std::atomic_uint64_t WriteCount{};
+	metrics::RequestStats m_AddChunkOps;
+	metrics::RequestStats m_FindChunkOps;
+	//	metrics::OperationTiming m_ContainChunkOps;
 
 	uint64_t m_LastScrubTime = 0;
 };
diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h
index 945f5d25a..1c8d79158 100644
--- a/src/zenstore/include/zenstore/cidstore.h
+++ b/src/zenstore/include/zenstore/cidstore.h
@@ -5,6 +5,7 @@
 #include "zenstore.h"
 
 #include <zencore/iohash.h>
+#include <zencore/stats.h>
 #include <zenstore/hashkeyset.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -31,9 +32,12 @@ struct CidStoreSize
 
 struct CidStoreStats
 {
-	uint64_t HitCount;
-	uint64_t MissCount;
-	uint64_t WriteCount;
+	uint64_t					  HitCount;
+	uint64_t					  MissCount;
+	uint64_t					  WriteCount;
+	metrics::RequestStatsSnapshot AddChunkOps;
+	metrics::RequestStatsSnapshot FindChunkOps;
+	//	metrics::RequestStatsSnapshot ContainChunkOps;
 };
 
 struct CidStoreConfiguration
-- 
cgit v1.2.3