oplog and cache stats (#244)

* basic oplog stats
* add GetValueStats to cache store
* RwLock::ExclusiveLockScope -> RwLock::SharedLockScope
* add rawhash and attachment count to CacheValueStats
* added cache-stats and project-stats commands
* add cast to make Mac overload detection happy
* fix accept type in cache-stats command
* Add options to project-stats command
* use resource paths for stats in project store
* use resource paths for stats in cache
* fix cache-info and project-info url discriminator
* more control over details$ output
* cleanup
* changelog

4 files changed, 397 insertions, 35 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 8539d9c16..90e905bf6 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -83,6 +83,7 @@ namespace {
 	static constinit std::string_view HttpZCacheUtilStartRecording	= "exec$/start-recording"sv;
 	static constinit std::string_view HttpZCacheUtilStopRecording	= "exec$/stop-recording"sv;
 	static constinit std::string_view HttpZCacheUtilReplayRecording = "exec$/replay-recording"sv;
+	static constinit std::string_view HttpZCacheDetailsPrefix		= "details$"sv;
 
 	struct HttpRequestData
 	{
@@ -366,6 +367,224 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx)
 }
 
 void
+HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
+{
+	std::string_view		 Key = Request.RelativeUri();
+	std::vector<std::string> Tokens;
+	uint32_t				 TokenCount = ForEachStrTok(Key, '/', [&Tokens](std::string_view Token) {
+		Tokens.push_back(std::string(Token));
+		return true;
+	});
+	std::string				 FilterNamespace;
+	std::string				 FilterBucket;
+	std::string				 FilterValue;
+	switch (TokenCount)
+	{
+		case 1:
+			break;
+		case 2:
+			{
+				FilterNamespace = Tokens[1];
+				if (FilterNamespace.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+			}
+			break;
+		case 3:
+			{
+				FilterNamespace = Tokens[1];
+				if (FilterNamespace.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+				FilterBucket = Tokens[2];
+				if (FilterBucket.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+			}
+			break;
+		case 4:
+			{
+				FilterNamespace = Tokens[1];
+				if (FilterNamespace.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+				FilterBucket = Tokens[2];
+				if (FilterBucket.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+				FilterValue = Tokens[3];
+				if (FilterValue.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+			}
+			break;
+		default:
+			return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+	}
+
+	HttpServerRequest::QueryParams Params			 = Request.GetQueryParams();
+	bool						   CSV				 = Params.GetValue("csv") == "true";
+	bool						   Details			 = Params.GetValue("details") == "true";
+	bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+	std::chrono::seconds NowSeconds	  = std::chrono::duration_cast<std::chrono::seconds>(GcClock::Now().time_since_epoch());
+	CacheValueDetails	 ValueDetails = m_CacheStore.GetValueDetails(FilterNamespace, FilterBucket, FilterValue);
+
+	if (CSV)
+	{
+		ExtendableStringBuilder<4096> CSVWriter;
+		if (AttachmentDetails)
+		{
+			CSVWriter << "Namespace, Bucket, Key, Cid, Size";
+		}
+		else if (Details)
+		{
+			CSVWriter << "Namespace, Bucket, Key, Size, RawSize, RawHash, ContentType, Age, AttachmentsCount, AttachmentsSize";
+		}
+		else
+		{
+			CSVWriter << "Namespace, Bucket, Key";
+		}
+		for (const auto& NamespaceIt : ValueDetails.Namespaces)
+		{
+			const std::string& Namespace = NamespaceIt.first;
+			for (const auto& BucketIt : NamespaceIt.second.Buckets)
+			{
+				const std::string& Bucket = BucketIt.first;
+				for (const auto& ValueIt : BucketIt.second.Values)
+				{
+					if (AttachmentDetails)
+					{
+						for (const IoHash& Hash : ValueIt.second.Attachments)
+						{
+							IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+							CSVWriter << "\r\n"
+									  << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << Hash.ToHexString()
+									  << ", " << gsl::narrow<uint64_t>(Payload.GetSize());
+						}
+					}
+					else if (Details)
+					{
+						std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast<std::chrono::seconds>(
+							GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch());
+						CSVWriter << "\r\n"
+								  << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << ValueIt.second.Size << ","
+								  << ValueIt.second.RawSize << "," << ValueIt.second.RawHash.ToHexString() << ", "
+								  << ToString(ValueIt.second.ContentType) << ", " << (NowSeconds.count() - LastAccessedSeconds.count())
+								  << ", " << gsl::narrow<uint64_t>(ValueIt.second.Attachments.size());
+						size_t AttachmentsSize = 0;
+						for (const IoHash& Hash : ValueIt.second.Attachments)
+						{
+							IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+							AttachmentsSize += Payload.GetSize();
+						}
+						CSVWriter << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
+					}
+					else
+					{
+						CSVWriter << "\r\n" << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString();
+					}
+				}
+			}
+		}
+		return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+	}
+	else
+	{
+		CbObjectWriter Cbo;
+		Cbo.BeginArray("namespaces");
+		{
+			for (const auto& NamespaceIt : ValueDetails.Namespaces)
+			{
+				const std::string& Namespace = NamespaceIt.first;
+				Cbo.BeginObject();
+				{
+					Cbo.AddString("name", Namespace);
+					Cbo.BeginArray("buckets");
+					{
+						for (const auto& BucketIt : NamespaceIt.second.Buckets)
+						{
+							const std::string& Bucket = BucketIt.first;
+							Cbo.BeginObject();
+							{
+								Cbo.AddString("name", Bucket);
+								Cbo.BeginArray("values");
+								{
+									for (const auto& ValueIt : BucketIt.second.Values)
+									{
+										std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast<std::chrono::seconds>(
+											GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch());
+										Cbo.BeginObject();
+										{
+											Cbo.AddHash("key", ValueIt.first);
+											if (Details)
+											{
+												Cbo.AddInteger("size", ValueIt.second.Size);
+												if (ValueIt.second.Size > 0 && ValueIt.second.RawSize != 0 &&
+													ValueIt.second.RawSize != ValueIt.second.Size)
+												{
+													Cbo.AddInteger("rawsize", ValueIt.second.RawSize);
+													Cbo.AddHash("rawhash", ValueIt.second.RawHash);
+												}
+												Cbo.AddString("contenttype", ToString(ValueIt.second.ContentType));
+												Cbo.AddInteger("age", NowSeconds.count() - LastAccessedSeconds.count());
+												if (ValueIt.second.Attachments.size() > 0)
+												{
+													if (AttachmentDetails)
+													{
+														Cbo.BeginArray("attachments");
+														{
+															for (const IoHash& Hash : ValueIt.second.Attachments)
+															{
+																Cbo.BeginObject();
+																Cbo.AddHash("cid", Hash);
+																IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+																Cbo.AddInteger("size", gsl::narrow<uint64_t>(Payload.GetSize()));
+																Cbo.EndObject();
+															}
+														}
+														Cbo.EndArray();
+													}
+													else
+													{
+														Cbo.AddInteger("attachmentcount",
+																	   gsl::narrow<uint64_t>(ValueIt.second.Attachments.size()));
+														size_t AttachmentsSize = 0;
+														for (const IoHash& Hash : ValueIt.second.Attachments)
+														{
+															IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+															AttachmentsSize += Payload.GetSize();
+														}
+														Cbo.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize));
+													}
+												}
+											}
+										}
+										Cbo.EndObject();
+									}
+								}
+								Cbo.EndArray();
+							}
+							Cbo.EndObject();
+						}
+					}
+					Cbo.EndArray();
+				}
+				Cbo.EndObject();
+			}
+		}
+		Cbo.EndArray();
+		Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+	}
+}
+
+void
 HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 {
 	metrics::OperationTiming::Scope $(m_HttpRequests);
@@ -409,6 +628,11 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 		Request.WriteResponse(HttpResponseCode::OK);
 		return;
 	}
+	if (Key.starts_with(HttpZCacheDetailsPrefix))
+	{
+		HandleDetailsRequest(Request);
+		return;
+	}
 
 	HttpRequestData RequestData;
 	if (!HttpRequestParseRelativeUri(Key, RequestData))
@@ -2784,27 +3008,39 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	const GcStorageSize CacheSize = m_CacheStore.StorageSize();
 
 	Cbo.BeginObject("cache");
-	Cbo.BeginObject("size");
-	Cbo << "disk" << CacheSize.DiskSize;
-	Cbo << "memory" << CacheSize.MemorySize;
-	Cbo.EndObject();
-	Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-	Cbo << "hits" << HitCount << "misses" << MissCount;
-	Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
-	Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
-	Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "disk" << CacheSize.DiskSize;
+			Cbo << "memory" << CacheSize.MemorySize;
+		}
+		Cbo.EndObject();
+
+		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+		Cbo << "hits" << HitCount << "misses" << MissCount;
+		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
+		Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+	}
 	Cbo.EndObject();
+
 	Cbo.BeginObject("upstream");
-	m_UpstreamCache.GetStatus(Cbo);
+	{
+		m_UpstreamCache.GetStatus(Cbo);
+	}
 	Cbo.EndObject();
 
 	Cbo.BeginObject("cid");
-	Cbo.BeginObject("size");
-	Cbo << "tiny" << CidSize.TinySize;
-	Cbo << "small" << CidSize.SmallSize;
-	Cbo << "large" << CidSize.LargeSize;
-	Cbo << "total" << CidSize.TotalSize;
-	Cbo.EndObject();
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "tiny" << CidSize.TinySize;
+			Cbo << "small" << CidSize.SmallSize;
+			Cbo << "large" << CidSize.LargeSize;
+			Cbo << "total" << CidSize.TotalSize;
+		}
+		Cbo.EndObject();
+	}
 	Cbo.EndObject();
 
 	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index e9c58c3d6..4e7b98ac9 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -110,6 +110,7 @@ private:
 	void HandleGetCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl);
 	void HandlePutCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl);
 	void HandleRpcRequest(HttpServerRequest& Request);
+	void HandleDetailsRequest(HttpServerRequest& Request);
 
 	CbPackage HandleRpcPutCacheRecords(const CbPackage& BatchRequest);
 	CbPackage HandleRpcGetCacheRecords(CbObjectView BatchRequest);
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 55af85ade..44574ae19 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -421,6 +421,12 @@ ZenCacheNamespace::GetBucketInfo(std::string_view Bucket) const
 	return Info;
 }
 
+CacheValueDetails::NamespaceDetails
+ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
+{
+	return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter);
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 ZenCacheMemoryLayer::ZenCacheMemoryLayer()
@@ -589,7 +595,7 @@ ZenCacheMemoryLayer::GetInfo() const
 std::optional<ZenCacheMemoryLayer::BucketInfo>
 ZenCacheMemoryLayer::GetBucketInfo(std::string_view Bucket) const
 {
-	RwLock::ExclusiveLockScope _(m_Lock);
+	RwLock::SharedLockScope _(m_Lock);
 
 	if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end())
 	{
@@ -1195,7 +1201,7 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const bool IsNew)
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey)
+ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const
 {
 	char HexString[sizeof(HashKey.Hash) * 2];
 	ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, HexString);
@@ -1212,7 +1218,7 @@ ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& H
 }
 
 IoBuffer
-ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc)
+ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc) const
 {
 	BlockStoreLocation Location = Loc.GetBlockLocation(m_PayloadAlignment);
 
@@ -1226,7 +1232,7 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc)
 }
 
 IoBuffer
-ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey)
+ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const
 {
 	ExtendablePathBuilder<256> DataFilePath;
 	BuildPath(DataFilePath, HashKey);
@@ -2036,6 +2042,50 @@ ZenCacheDiskLayer::CacheBucket::EntryCount() const
 	return static_cast<uint64_t>(m_Index.size());
 }
 
+CacheValueDetails::ValueDetails
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, size_t Index) const
+{
+	std::vector<IoHash>	 Attachments;
+	const BucketPayload& Payload = m_Payloads[Index];
+	if (Payload.Location.IsFlagSet(DiskLocation::kStructured))
+	{
+		IoBuffer Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile) ? GetStandaloneCacheValue(Payload.Location, Key)
+																				   : GetInlineCacheValue(Payload.Location);
+		CbObject Obj(SharedBuffer{Value});
+		Obj.IterateAttachments([&Attachments](CbFieldView Field) { Attachments.emplace_back(Field.AsAttachment()); });
+	}
+	return CacheValueDetails::ValueDetails{.Size		= Payload.Location.Size(),
+										   .RawSize		= Payload.RawSize,
+										   .RawHash		= Payload.RawHash,
+										   .LastAccess	= m_AccessTimes[Index],
+										   .Attachments = std::move(Attachments),
+										   .ContentType = Payload.Location.GetContentType()};
+}
+
+CacheValueDetails::BucketDetails
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilter) const
+{
+	CacheValueDetails::BucketDetails Details;
+	RwLock::SharedLockScope			 _(m_IndexLock);
+	if (ValueFilter.empty())
+	{
+		Details.Values.reserve(m_Index.size());
+		for (const auto& It : m_Index)
+		{
+			Details.Values.insert_or_assign(It.first, GetValueDetails(It.first, It.second));
+		}
+	}
+	else
+	{
+		IoHash Key = IoHash::FromHexString(ValueFilter);
+		if (auto It = m_Index.find(Key); It != m_Index.end())
+		{
+			Details.Values.insert_or_assign(It->first, GetValueDetails(It->first, It->second));
+		}
+	}
+	return Details;
+}
+
 void
 ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx)
 {
@@ -2505,7 +2555,7 @@ ZenCacheDiskLayer::GetInfo() const
 std::optional<ZenCacheDiskLayer::BucketInfo>
 ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const
 {
-	RwLock::ExclusiveLockScope _(m_Lock);
+	RwLock::SharedLockScope _(m_Lock);
 
 	if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end())
 	{
@@ -2514,6 +2564,26 @@ ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const
 	return {};
 }
 
+CacheValueDetails::NamespaceDetails
+ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
+{
+	RwLock::SharedLockScope				_(m_Lock);
+	CacheValueDetails::NamespaceDetails Details;
+	if (BucketFilter.empty())
+	{
+		Details.Buckets.reserve(BucketFilter.empty() ? m_Buckets.size() : 1);
+		for (auto& Kv : m_Buckets)
+		{
+			Details.Buckets[Kv.first] = Kv.second->GetValueDetails(ValueFilter);
+		}
+	}
+	else if (auto It = m_Buckets.find(std::string(BucketFilter)); It != m_Buckets.end())
+	{
+		Details.Buckets[It->first] = It->second->GetValueDetails(ValueFilter);
+	}
+	return Details;
+}
+
 //////////////////////////// ZenCacheStore
 
 static constexpr std::string_view UE4DDCNamespaceName = "ue4.ddc";
@@ -2568,6 +2638,7 @@ ZenCacheStore::Get(std::string_view Namespace, std::string_view Bucket, const Io
 		return Store->Get(Bucket, HashKey, OutValue);
 	}
 	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get, bucket '{}', key '{}'", Namespace, Bucket, HashKey.ToHexString());
+
 	return false;
 }
 
@@ -2619,6 +2690,25 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Scrub(Ctx); });
 }
 
+CacheValueDetails
+ZenCacheStore::GetValueDetails(const std::string_view NamespaceFilter,
+							   const std::string_view BucketFilter,
+							   const std::string_view ValueFilter) const
+{
+	CacheValueDetails Details;
+	if (NamespaceFilter.empty())
+	{
+		IterateNamespaces([&](std::string_view Namespace, ZenCacheNamespace& Store) {
+			Details.Namespaces[std::string(Namespace)] = Store.GetValueDetails(BucketFilter, ValueFilter);
+		});
+	}
+	else if (const ZenCacheNamespace* Store = FindNamespace(NamespaceFilter); Store != nullptr)
+	{
+		Details.Namespaces[std::string(NamespaceFilter)] = Store->GetValueDetails(BucketFilter, ValueFilter);
+	}
+	return Details;
+}
+
 ZenCacheNamespace*
 ZenCacheStore::GetNamespace(std::string_view Namespace)
 {
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index fe83f3c97..2ef4c4dcb 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -67,6 +67,31 @@ struct ZenCacheValue
 	IoHash	 RawHash = IoHash::Zero;
 };
 
+struct CacheValueDetails
+{
+	struct ValueDetails
+	{
+		uint64_t			Size;
+		uint64_t			RawSize;
+		IoHash				RawHash;
+		GcClock::Tick		LastAccess{};
+		std::vector<IoHash> Attachments;
+		ZenContentType		ContentType;
+	};
+
+	struct BucketDetails
+	{
+		std::unordered_map<IoHash, ValueDetails, IoHash::Hasher> Values;
+	};
+
+	struct NamespaceDetails
+	{
+		std::unordered_map<std::string, BucketDetails> Buckets;
+	};
+
+	std::unordered_map<std::string, NamespaceDetails> Namespaces;
+};
+
 //////////////////////////////////////////////////////////////////////////
 
 #pragma pack(push)
@@ -301,6 +326,8 @@ public:
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
 
+	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
+
 private:
 	/** A cache bucket manages a single directory containing
 		metadata and data for that bucket
@@ -323,6 +350,8 @@ private:
 		inline uint64_t TotalSize() const { return m_TotalStandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(); }
 		uint64_t		EntryCount() const;
 
+		CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const;
+
 	private:
 		const uint64_t MaxBlockSize		  = 1ull << 30;
 		uint64_t	   m_PayloadAlignment = 1ull << 4;
@@ -360,18 +389,18 @@ private:
 
 		std::atomic_uint64_t m_TotalStandaloneSize{};
 
-		void	 BuildPath(PathBuilderBase& Path, const IoHash& HashKey);
-		void	 PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
-		IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey);
-		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
-		IoBuffer GetInlineCacheValue(const DiskLocation& Loc);
-		void	 MakeIndexSnapshot();
-		uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
-		uint64_t ReadLogV2(const std::filesystem::path& LogPath, uint64_t LogPosition);
-		uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
-		void	 OpenLog(const bool IsNew);
-		void	 SaveManifest();
-
+		void							BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
+		void							PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+		IoBuffer						GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const;
+		void							PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+		IoBuffer						GetInlineCacheValue(const DiskLocation& Loc) const;
+		void							MakeIndexSnapshot();
+		uint64_t						ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
+		uint64_t						ReadLogV2(const std::filesystem::path& LogPath, uint64_t LogPosition);
+		uint64_t						ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
+		void							OpenLog(const bool IsNew);
+		void							SaveManifest();
+		CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, size_t Index) const;
 		// These locks are here to avoid contention on file creation, therefore it's sufficient
 		// that we take the same lock for the same hash
 		//
@@ -379,8 +408,8 @@ private:
 		// but we don't currently access them at particularly high frequency so it should not be
 		// an issue in practice
 
-		RwLock		   m_ShardedLocks[256];
-		inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; }
+		mutable RwLock m_ShardedLocks[256];
+		inline RwLock& LockForHash(const IoHash& Hash) const { return m_ShardedLocks[Hash.Hash[19]]; }
 	};
 
 	std::filesystem::path										  m_RootDir;
@@ -429,6 +458,8 @@ public:
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
 
+	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
+
 private:
 	std::filesystem::path m_RootDir;
 	ZenCacheMemoryLayer	  m_MemLayer;
@@ -476,6 +507,10 @@ public:
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
 
+	CacheValueDetails GetValueDetails(const std::string_view NamespaceFilter,
+									  const std::string_view BucketFilter,
+									  const std::string_view ValueFilter) const;
+
 	GcStorageSize StorageSize() const;
 	//	const Configuration& GetConfiguration() const { return m_Configuration; }