oplog and cache stats (#244)

* basic oplog stats
* add GetValueStats to cache store
* RwLock::ExclusiveLockScope -> RwLock::SharedLockScope
* add rawhash and attachment count to CacheValueStats
* added cache-stats and project-stats commands
* add cast to make Mac overload detection happy
* fix accept type in cache-stats command
* Add options to project-stats command
* use resource paths for stats in project store
* use resource paths for stats in cache
* fix cache-info and project-info url discriminator
* more control over details$ output
* cleanup
* changelog

7 files changed, 927 insertions, 39 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 8539d9c16..90e905bf6 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -83,6 +83,7 @@ namespace {
 	static constinit std::string_view HttpZCacheUtilStartRecording	= "exec$/start-recording"sv;
 	static constinit std::string_view HttpZCacheUtilStopRecording	= "exec$/stop-recording"sv;
 	static constinit std::string_view HttpZCacheUtilReplayRecording = "exec$/replay-recording"sv;
+	static constinit std::string_view HttpZCacheDetailsPrefix		= "details$"sv;
 
 	struct HttpRequestData
 	{
@@ -366,6 +367,224 @@ HttpStructuredCacheService::Scrub(ScrubContext& Ctx)
 }
 
 void
+HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
+{
+	std::string_view		 Key = Request.RelativeUri();
+	std::vector<std::string> Tokens;
+	uint32_t				 TokenCount = ForEachStrTok(Key, '/', [&Tokens](std::string_view Token) {
+		Tokens.push_back(std::string(Token));
+		return true;
+	});
+	std::string				 FilterNamespace;
+	std::string				 FilterBucket;
+	std::string				 FilterValue;
+	switch (TokenCount)
+	{
+		case 1:
+			break;
+		case 2:
+			{
+				FilterNamespace = Tokens[1];
+				if (FilterNamespace.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+			}
+			break;
+		case 3:
+			{
+				FilterNamespace = Tokens[1];
+				if (FilterNamespace.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+				FilterBucket = Tokens[2];
+				if (FilterBucket.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+			}
+			break;
+		case 4:
+			{
+				FilterNamespace = Tokens[1];
+				if (FilterNamespace.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+				FilterBucket = Tokens[2];
+				if (FilterBucket.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+				FilterValue = Tokens[3];
+				if (FilterValue.empty())
+				{
+					return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+				}
+			}
+			break;
+		default:
+			return Request.WriteResponse(HttpResponseCode::BadRequest);	 // invalid URL
+	}
+
+	HttpServerRequest::QueryParams Params			 = Request.GetQueryParams();
+	bool						   CSV				 = Params.GetValue("csv") == "true";
+	bool						   Details			 = Params.GetValue("details") == "true";
+	bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+	std::chrono::seconds NowSeconds	  = std::chrono::duration_cast<std::chrono::seconds>(GcClock::Now().time_since_epoch());
+	CacheValueDetails	 ValueDetails = m_CacheStore.GetValueDetails(FilterNamespace, FilterBucket, FilterValue);
+
+	if (CSV)
+	{
+		ExtendableStringBuilder<4096> CSVWriter;
+		if (AttachmentDetails)
+		{
+			CSVWriter << "Namespace, Bucket, Key, Cid, Size";
+		}
+		else if (Details)
+		{
+			CSVWriter << "Namespace, Bucket, Key, Size, RawSize, RawHash, ContentType, Age, AttachmentsCount, AttachmentsSize";
+		}
+		else
+		{
+			CSVWriter << "Namespace, Bucket, Key";
+		}
+		for (const auto& NamespaceIt : ValueDetails.Namespaces)
+		{
+			const std::string& Namespace = NamespaceIt.first;
+			for (const auto& BucketIt : NamespaceIt.second.Buckets)
+			{
+				const std::string& Bucket = BucketIt.first;
+				for (const auto& ValueIt : BucketIt.second.Values)
+				{
+					if (AttachmentDetails)
+					{
+						for (const IoHash& Hash : ValueIt.second.Attachments)
+						{
+							IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+							CSVWriter << "\r\n"
+									  << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << Hash.ToHexString()
+									  << ", " << gsl::narrow<uint64_t>(Payload.GetSize());
+						}
+					}
+					else if (Details)
+					{
+						std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast<std::chrono::seconds>(
+							GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch());
+						CSVWriter << "\r\n"
+								  << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << ValueIt.second.Size << ","
+								  << ValueIt.second.RawSize << "," << ValueIt.second.RawHash.ToHexString() << ", "
+								  << ToString(ValueIt.second.ContentType) << ", " << (NowSeconds.count() - LastAccessedSeconds.count())
+								  << ", " << gsl::narrow<uint64_t>(ValueIt.second.Attachments.size());
+						size_t AttachmentsSize = 0;
+						for (const IoHash& Hash : ValueIt.second.Attachments)
+						{
+							IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+							AttachmentsSize += Payload.GetSize();
+						}
+						CSVWriter << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
+					}
+					else
+					{
+						CSVWriter << "\r\n" << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString();
+					}
+				}
+			}
+		}
+		return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+	}
+	else
+	{
+		CbObjectWriter Cbo;
+		Cbo.BeginArray("namespaces");
+		{
+			for (const auto& NamespaceIt : ValueDetails.Namespaces)
+			{
+				const std::string& Namespace = NamespaceIt.first;
+				Cbo.BeginObject();
+				{
+					Cbo.AddString("name", Namespace);
+					Cbo.BeginArray("buckets");
+					{
+						for (const auto& BucketIt : NamespaceIt.second.Buckets)
+						{
+							const std::string& Bucket = BucketIt.first;
+							Cbo.BeginObject();
+							{
+								Cbo.AddString("name", Bucket);
+								Cbo.BeginArray("values");
+								{
+									for (const auto& ValueIt : BucketIt.second.Values)
+									{
+										std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast<std::chrono::seconds>(
+											GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch());
+										Cbo.BeginObject();
+										{
+											Cbo.AddHash("key", ValueIt.first);
+											if (Details)
+											{
+												Cbo.AddInteger("size", ValueIt.second.Size);
+												if (ValueIt.second.Size > 0 && ValueIt.second.RawSize != 0 &&
+													ValueIt.second.RawSize != ValueIt.second.Size)
+												{
+													Cbo.AddInteger("rawsize", ValueIt.second.RawSize);
+													Cbo.AddHash("rawhash", ValueIt.second.RawHash);
+												}
+												Cbo.AddString("contenttype", ToString(ValueIt.second.ContentType));
+												Cbo.AddInteger("age", NowSeconds.count() - LastAccessedSeconds.count());
+												if (ValueIt.second.Attachments.size() > 0)
+												{
+													if (AttachmentDetails)
+													{
+														Cbo.BeginArray("attachments");
+														{
+															for (const IoHash& Hash : ValueIt.second.Attachments)
+															{
+																Cbo.BeginObject();
+																Cbo.AddHash("cid", Hash);
+																IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+																Cbo.AddInteger("size", gsl::narrow<uint64_t>(Payload.GetSize()));
+																Cbo.EndObject();
+															}
+														}
+														Cbo.EndArray();
+													}
+													else
+													{
+														Cbo.AddInteger("attachmentcount",
+																	   gsl::narrow<uint64_t>(ValueIt.second.Attachments.size()));
+														size_t AttachmentsSize = 0;
+														for (const IoHash& Hash : ValueIt.second.Attachments)
+														{
+															IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
+															AttachmentsSize += Payload.GetSize();
+														}
+														Cbo.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize));
+													}
+												}
+											}
+										}
+										Cbo.EndObject();
+									}
+								}
+								Cbo.EndArray();
+							}
+							Cbo.EndObject();
+						}
+					}
+					Cbo.EndArray();
+				}
+				Cbo.EndObject();
+			}
+		}
+		Cbo.EndArray();
+		Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+	}
+}
+
+void
 HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 {
 	metrics::OperationTiming::Scope $(m_HttpRequests);
@@ -409,6 +628,11 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 		Request.WriteResponse(HttpResponseCode::OK);
 		return;
 	}
+	if (Key.starts_with(HttpZCacheDetailsPrefix))
+	{
+		HandleDetailsRequest(Request);
+		return;
+	}
 
 	HttpRequestData RequestData;
 	if (!HttpRequestParseRelativeUri(Key, RequestData))
@@ -2784,27 +3008,39 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
 	const GcStorageSize CacheSize = m_CacheStore.StorageSize();
 
 	Cbo.BeginObject("cache");
-	Cbo.BeginObject("size");
-	Cbo << "disk" << CacheSize.DiskSize;
-	Cbo << "memory" << CacheSize.MemorySize;
-	Cbo.EndObject();
-	Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
-	Cbo << "hits" << HitCount << "misses" << MissCount;
-	Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
-	Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
-	Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "disk" << CacheSize.DiskSize;
+			Cbo << "memory" << CacheSize.MemorySize;
+		}
+		Cbo.EndObject();
+
+		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+		Cbo << "hits" << HitCount << "misses" << MissCount;
+		Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
+		Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+		Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+	}
 	Cbo.EndObject();
+
 	Cbo.BeginObject("upstream");
-	m_UpstreamCache.GetStatus(Cbo);
+	{
+		m_UpstreamCache.GetStatus(Cbo);
+	}
 	Cbo.EndObject();
 
 	Cbo.BeginObject("cid");
-	Cbo.BeginObject("size");
-	Cbo << "tiny" << CidSize.TinySize;
-	Cbo << "small" << CidSize.SmallSize;
-	Cbo << "large" << CidSize.LargeSize;
-	Cbo << "total" << CidSize.TotalSize;
-	Cbo.EndObject();
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "tiny" << CidSize.TinySize;
+			Cbo << "small" << CidSize.SmallSize;
+			Cbo << "large" << CidSize.LargeSize;
+			Cbo << "total" << CidSize.TotalSize;
+		}
+		Cbo.EndObject();
+	}
 	Cbo.EndObject();
 
 	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index e9c58c3d6..4e7b98ac9 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -110,6 +110,7 @@ private:
 	void HandleGetCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl);
 	void HandlePutCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl);
 	void HandleRpcRequest(HttpServerRequest& Request);
+	void HandleDetailsRequest(HttpServerRequest& Request);
 
 	CbPackage HandleRpcPutCacheRecords(const CbPackage& BatchRequest);
 	CbPackage HandleRpcGetCacheRecords(CbObjectView BatchRequest);
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 55af85ade..44574ae19 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -421,6 +421,12 @@ ZenCacheNamespace::GetBucketInfo(std::string_view Bucket) const
 	return Info;
 }
 
+CacheValueDetails::NamespaceDetails
+ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
+{
+	return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter);
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 ZenCacheMemoryLayer::ZenCacheMemoryLayer()
@@ -589,7 +595,7 @@ ZenCacheMemoryLayer::GetInfo() const
 std::optional<ZenCacheMemoryLayer::BucketInfo>
 ZenCacheMemoryLayer::GetBucketInfo(std::string_view Bucket) const
 {
-	RwLock::ExclusiveLockScope _(m_Lock);
+	RwLock::SharedLockScope _(m_Lock);
 
 	if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end())
 	{
@@ -1195,7 +1201,7 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const bool IsNew)
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey)
+ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const
 {
 	char HexString[sizeof(HashKey.Hash) * 2];
 	ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, HexString);
@@ -1212,7 +1218,7 @@ ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& H
 }
 
 IoBuffer
-ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc)
+ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc) const
 {
 	BlockStoreLocation Location = Loc.GetBlockLocation(m_PayloadAlignment);
 
@@ -1226,7 +1232,7 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc)
 }
 
 IoBuffer
-ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey)
+ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const
 {
 	ExtendablePathBuilder<256> DataFilePath;
 	BuildPath(DataFilePath, HashKey);
@@ -2036,6 +2042,50 @@ ZenCacheDiskLayer::CacheBucket::EntryCount() const
 	return static_cast<uint64_t>(m_Index.size());
 }
 
+CacheValueDetails::ValueDetails
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, size_t Index) const
+{
+	std::vector<IoHash>	 Attachments;
+	const BucketPayload& Payload = m_Payloads[Index];
+	if (Payload.Location.IsFlagSet(DiskLocation::kStructured))
+	{
+		IoBuffer Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile) ? GetStandaloneCacheValue(Payload.Location, Key)
+																				   : GetInlineCacheValue(Payload.Location);
+		CbObject Obj(SharedBuffer{Value});
+		Obj.IterateAttachments([&Attachments](CbFieldView Field) { Attachments.emplace_back(Field.AsAttachment()); });
+	}
+	return CacheValueDetails::ValueDetails{.Size		= Payload.Location.Size(),
+										   .RawSize		= Payload.RawSize,
+										   .RawHash		= Payload.RawHash,
+										   .LastAccess	= m_AccessTimes[Index],
+										   .Attachments = std::move(Attachments),
+										   .ContentType = Payload.Location.GetContentType()};
+}
+
+CacheValueDetails::BucketDetails
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilter) const
+{
+	CacheValueDetails::BucketDetails Details;
+	RwLock::SharedLockScope			 _(m_IndexLock);
+	if (ValueFilter.empty())
+	{
+		Details.Values.reserve(m_Index.size());
+		for (const auto& It : m_Index)
+		{
+			Details.Values.insert_or_assign(It.first, GetValueDetails(It.first, It.second));
+		}
+	}
+	else
+	{
+		IoHash Key = IoHash::FromHexString(ValueFilter);
+		if (auto It = m_Index.find(Key); It != m_Index.end())
+		{
+			Details.Values.insert_or_assign(It->first, GetValueDetails(It->first, It->second));
+		}
+	}
+	return Details;
+}
+
 void
 ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx)
 {
@@ -2505,7 +2555,7 @@ ZenCacheDiskLayer::GetInfo() const
 std::optional<ZenCacheDiskLayer::BucketInfo>
 ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const
 {
-	RwLock::ExclusiveLockScope _(m_Lock);
+	RwLock::SharedLockScope _(m_Lock);
 
 	if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end())
 	{
@@ -2514,6 +2564,26 @@ ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const
 	return {};
 }
 
+CacheValueDetails::NamespaceDetails
+ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
+{
+	RwLock::SharedLockScope				_(m_Lock);
+	CacheValueDetails::NamespaceDetails Details;
+	if (BucketFilter.empty())
+	{
+		Details.Buckets.reserve(BucketFilter.empty() ? m_Buckets.size() : 1);
+		for (auto& Kv : m_Buckets)
+		{
+			Details.Buckets[Kv.first] = Kv.second->GetValueDetails(ValueFilter);
+		}
+	}
+	else if (auto It = m_Buckets.find(std::string(BucketFilter)); It != m_Buckets.end())
+	{
+		Details.Buckets[It->first] = It->second->GetValueDetails(ValueFilter);
+	}
+	return Details;
+}
+
 //////////////////////////// ZenCacheStore
 
 static constexpr std::string_view UE4DDCNamespaceName = "ue4.ddc";
@@ -2568,6 +2638,7 @@ ZenCacheStore::Get(std::string_view Namespace, std::string_view Bucket, const Io
 		return Store->Get(Bucket, HashKey, OutValue);
 	}
 	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get, bucket '{}', key '{}'", Namespace, Bucket, HashKey.ToHexString());
+
 	return false;
 }
 
@@ -2619,6 +2690,25 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Scrub(Ctx); });
 }
 
+CacheValueDetails
+ZenCacheStore::GetValueDetails(const std::string_view NamespaceFilter,
+							   const std::string_view BucketFilter,
+							   const std::string_view ValueFilter) const
+{
+	CacheValueDetails Details;
+	if (NamespaceFilter.empty())
+	{
+		IterateNamespaces([&](std::string_view Namespace, ZenCacheNamespace& Store) {
+			Details.Namespaces[std::string(Namespace)] = Store.GetValueDetails(BucketFilter, ValueFilter);
+		});
+	}
+	else if (const ZenCacheNamespace* Store = FindNamespace(NamespaceFilter); Store != nullptr)
+	{
+		Details.Namespaces[std::string(NamespaceFilter)] = Store->GetValueDetails(BucketFilter, ValueFilter);
+	}
+	return Details;
+}
+
 ZenCacheNamespace*
 ZenCacheStore::GetNamespace(std::string_view Namespace)
 {
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index fe83f3c97..2ef4c4dcb 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -67,6 +67,31 @@ struct ZenCacheValue
 	IoHash	 RawHash = IoHash::Zero;
 };
 
+struct CacheValueDetails
+{
+	struct ValueDetails
+	{
+		uint64_t			Size;
+		uint64_t			RawSize;
+		IoHash				RawHash;
+		GcClock::Tick		LastAccess{};
+		std::vector<IoHash> Attachments;
+		ZenContentType		ContentType;
+	};
+
+	struct BucketDetails
+	{
+		std::unordered_map<IoHash, ValueDetails, IoHash::Hasher> Values;
+	};
+
+	struct NamespaceDetails
+	{
+		std::unordered_map<std::string, BucketDetails> Buckets;
+	};
+
+	std::unordered_map<std::string, NamespaceDetails> Namespaces;
+};
+
 //////////////////////////////////////////////////////////////////////////
 
 #pragma pack(push)
@@ -301,6 +326,8 @@ public:
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
 
+	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
+
 private:
 	/** A cache bucket manages a single directory containing
 		metadata and data for that bucket
@@ -323,6 +350,8 @@ private:
 		inline uint64_t TotalSize() const { return m_TotalStandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(); }
 		uint64_t		EntryCount() const;
 
+		CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const;
+
 	private:
 		const uint64_t MaxBlockSize		  = 1ull << 30;
 		uint64_t	   m_PayloadAlignment = 1ull << 4;
@@ -360,18 +389,18 @@ private:
 
 		std::atomic_uint64_t m_TotalStandaloneSize{};
 
-		void	 BuildPath(PathBuilderBase& Path, const IoHash& HashKey);
-		void	 PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
-		IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey);
-		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
-		IoBuffer GetInlineCacheValue(const DiskLocation& Loc);
-		void	 MakeIndexSnapshot();
-		uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
-		uint64_t ReadLogV2(const std::filesystem::path& LogPath, uint64_t LogPosition);
-		uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
-		void	 OpenLog(const bool IsNew);
-		void	 SaveManifest();
-
+		void							BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
+		void							PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+		IoBuffer						GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const;
+		void							PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+		IoBuffer						GetInlineCacheValue(const DiskLocation& Loc) const;
+		void							MakeIndexSnapshot();
+		uint64_t						ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
+		uint64_t						ReadLogV2(const std::filesystem::path& LogPath, uint64_t LogPosition);
+		uint64_t						ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
+		void							OpenLog(const bool IsNew);
+		void							SaveManifest();
+		CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, size_t Index) const;
 		// These locks are here to avoid contention on file creation, therefore it's sufficient
 		// that we take the same lock for the same hash
 		//
@@ -379,8 +408,8 @@ private:
 		// but we don't currently access them at particularly high frequency so it should not be
 		// an issue in practice
 
-		RwLock		   m_ShardedLocks[256];
-		inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; }
+		mutable RwLock m_ShardedLocks[256];
+		inline RwLock& LockForHash(const IoHash& Hash) const { return m_ShardedLocks[Hash.Hash[19]]; }
 	};
 
 	std::filesystem::path										  m_RootDir;
@@ -429,6 +458,8 @@ public:
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
 
+	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
+
 private:
 	std::filesystem::path m_RootDir;
 	ZenCacheMemoryLayer	  m_MemLayer;
@@ -476,6 +507,10 @@ public:
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
 
+	CacheValueDetails GetValueDetails(const std::string_view NamespaceFilter,
+									  const std::string_view BucketFilter,
+									  const std::string_view ValueFilter) const;
+
 	GcStorageSize StorageSize() const;
 	//	const Configuration& GetConfiguration() const { return m_Configuration; }
 
diff --git a/zenserver/projectstore/projectstore.cpp b/zenserver/projectstore/projectstore.cpp
index 5aa8cad26..db5cae503 100644
--- a/zenserver/projectstore/projectstore.cpp
+++ b/zenserver/projectstore/projectstore.cpp
@@ -201,6 +201,204 @@ namespace {
 									  : fmt::format("{}. Reason: '{}'", Result.Text, Result.Reason)};
 	}
 
+	void CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter)
+	{
+		if (AttachmentDetails)
+		{
+			CSVWriter << "Project, Oplog, LSN, Key, Cid, Size";
+		}
+		else if (Details)
+		{
+			CSVWriter << "Project, Oplog, LSN, Key, Size, AttachmentCount, AttachmentsSize";
+		}
+		else
+		{
+			CSVWriter << "Project, Oplog, Key";
+		}
+	}
+
+	void CSVWriteOp(CidStore&		   CidStore,
+					std::string_view   ProjectId,
+					std::string_view   OplogId,
+					bool			   Details,
+					bool			   AttachmentDetails,
+					int				   LSN,
+					const Oid&		   Key,
+					CbObject		   Op,
+					StringBuilderBase& CSVWriter)
+	{
+		StringBuilder<32> KeyStringBuilder;
+		Key.ToString(KeyStringBuilder);
+		const std::string_view KeyString = KeyStringBuilder.ToView();
+
+		SharedBuffer Buffer = Op.GetBuffer();
+		if (AttachmentDetails)
+		{
+			Op.IterateAttachments([&CidStore, &CSVWriter, &ProjectId, &OplogId, LSN, &KeyString](CbFieldView FieldView) {
+				const IoHash AttachmentHash = FieldView.AsAttachment();
+				IoBuffer	 Attachment		= CidStore.FindChunkByCid(AttachmentHash);
+				CSVWriter << "\r\n"
+						  << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << AttachmentHash.ToHexString()
+						  << ", " << gsl::narrow<uint64_t>(Attachment.GetSize());
+			});
+		}
+		else if (Details)
+		{
+			uint64_t AttachmentCount = 0;
+			size_t	 AttachmentsSize = 0;
+			Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
+				const IoHash AttachmentHash = FieldView.AsAttachment();
+				AttachmentCount++;
+				IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+				AttachmentsSize += Attachment.GetSize();
+			});
+			CSVWriter << "\r\n"
+					  << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Buffer.GetSize())
+					  << ", " << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
+		}
+		else
+		{
+			CSVWriter << "\r\n" << ProjectId << ", " << OplogId << ", " << KeyString;
+		}
+	};
+
+	void CbWriteOp(CidStore&	   CidStore,
+				   bool			   Details,
+				   bool			   OpDetails,
+				   bool			   AttachmentDetails,
+				   int			   LSN,
+				   const Oid&	   Key,
+				   CbObject		   Op,
+				   CbObjectWriter& CbWriter)
+	{
+		CbWriter.BeginObject();
+		{
+			SharedBuffer Buffer = Op.GetBuffer();
+			CbWriter.AddObjectId("key", Key);
+			if (Details)
+			{
+				CbWriter.AddInteger("lsn", LSN);
+				CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Buffer.GetSize()));
+			}
+			if (AttachmentDetails)
+			{
+				CbWriter.BeginArray("attachments");
+				Op.IterateAttachments([&CidStore, &CbWriter](CbFieldView FieldView) {
+					const IoHash AttachmentHash = FieldView.AsAttachment();
+					CbWriter.BeginObject();
+					{
+						IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+						CbWriter.AddString("cid", AttachmentHash.ToHexString());
+						CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Attachment.GetSize()));
+					}
+					CbWriter.EndObject();
+				});
+				CbWriter.EndArray();
+			}
+			else if (Details)
+			{
+				uint64_t AttachmentCount = 0;
+				size_t	 AttachmentsSize = 0;
+				Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) {
+					const IoHash AttachmentHash = FieldView.AsAttachment();
+					AttachmentCount++;
+					IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash);
+					AttachmentsSize += Attachment.GetSize();
+				});
+				if (AttachmentCount > 0)
+				{
+					CbWriter.AddInteger("attachments", AttachmentCount);
+					CbWriter.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize));
+				}
+			}
+			if (OpDetails)
+			{
+				CbWriter.BeginObject("op");
+				for (const CbFieldView& Field : Op)
+				{
+					if (!Field.HasName())
+					{
+						CbWriter.AddField(Field);
+						continue;
+					}
+					std::string_view FieldName = Field.GetName();
+					CbWriter.AddField(FieldName, Field);
+				}
+				CbWriter.EndObject();
+			}
+		}
+		CbWriter.EndObject();
+	};
+
+	void CbWriteOplogOps(CidStore&			  CidStore,
+						 ProjectStore::Oplog& Oplog,
+						 bool				  Details,
+						 bool				  OpDetails,
+						 bool				  AttachmentDetails,
+						 CbObjectWriter&	  Cbo)
+	{
+		Cbo.BeginArray("ops");
+		{
+			Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
+				CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo);
+			});
+		}
+		Cbo.EndArray();
+	}
+
+	void CbWriteOplog(CidStore&			   CidStore,
+					  ProjectStore::Oplog& Oplog,
+					  bool				   Details,
+					  bool				   OpDetails,
+					  bool				   AttachmentDetails,
+					  CbObjectWriter&	   Cbo)
+	{
+		Cbo.BeginObject();
+		{
+			Cbo.AddString("name", Oplog.OplogId());
+			CbWriteOplogOps(CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+		}
+		Cbo.EndObject();
+	}
+
+	void CbWriteOplogs(CidStore&				CidStore,
+					   ProjectStore::Project&	Project,
+					   std::vector<std::string> OpLogs,
+					   bool						Details,
+					   bool						OpDetails,
+					   bool						AttachmentDetails,
+					   CbObjectWriter&			Cbo)
+	{
+		Cbo.BeginArray("oplogs");
+		{
+			for (const std::string& OpLogId : OpLogs)
+			{
+				ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId);
+				if (Oplog != nullptr)
+				{
+					CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+				}
+			}
+		}
+		Cbo.EndArray();
+	}
+
+	void CbWriteProject(CidStore&				 CidStore,
+						ProjectStore::Project&	 Project,
+						std::vector<std::string> OpLogs,
+						bool					 Details,
+						bool					 OpDetails,
+						bool					 AttachmentDetails,
+						CbObjectWriter&			 Cbo)
+	{
+		Cbo.BeginObject();
+		{
+			Cbo.AddString("name", Project.Identifier);
+			CbWriteOplogs(CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+		}
+		Cbo.EndObject();
+	}
+
 }  // namespace
 
 //////////////////////////////////////////////////////////////////////////
@@ -681,6 +879,69 @@ ProjectStore::Oplog::IterateOplog(std::function<void(CbObject)>&& Handler)
 	m_Storage->ReplayLog(Entries, [&](CbObject Op) { Handler(Op); });
 }
 
+void
+ProjectStore::Oplog::IterateOplogWithKey(std::function<void(int, const Oid&, CbObject)>&& Handler)
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+	if (!m_Storage)
+	{
+		return;
+	}
+
+	std::vector<size_t>			   EntryIndexes;
+	std::vector<OplogEntryAddress> Entries;
+	std::vector<Oid>			   Keys;
+	std::vector<int>			   LSNs;
+	Entries.reserve(m_LatestOpMap.size());
+	EntryIndexes.reserve(m_LatestOpMap.size());
+	Keys.reserve(m_LatestOpMap.size());
+	LSNs.reserve(m_LatestOpMap.size());
+
+	for (const auto& Kv : m_LatestOpMap)
+	{
+		const auto AddressEntry = m_OpAddressMap.find(Kv.second);
+		ZEN_ASSERT(AddressEntry != m_OpAddressMap.end());
+
+		Entries.push_back(AddressEntry->second);
+		Keys.push_back(Kv.first);
+		LSNs.push_back(Kv.second);
+		EntryIndexes.push_back(EntryIndexes.size());
+	}
+
+	std::sort(EntryIndexes.begin(), EntryIndexes.end(), [&Entries](const size_t& Lhs, const size_t& Rhs) {
+		const OplogEntryAddress& LhsEntry = Entries[Lhs];
+		const OplogEntryAddress& RhsEntry = Entries[Rhs];
+		return LhsEntry.Offset < RhsEntry.Offset;
+	});
+	std::vector<OplogEntryAddress> SortedEntries;
+	SortedEntries.reserve(EntryIndexes.size());
+	for (size_t Index : EntryIndexes)
+	{
+		SortedEntries.push_back(Entries[Index]);
+	}
+
+	size_t EntryIndex = 0;
+	m_Storage->ReplayLog(SortedEntries, [&](CbObject Op) {
+		Handler(LSNs[EntryIndex], Keys[EntryIndex], Op);
+		EntryIndex++;
+	});
+}
+
+int
+ProjectStore::Oplog::GetOpIndexByKey(const Oid& Key)
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+	if (!m_Storage)
+	{
+		return {};
+	}
+	if (const auto LatestOp = m_LatestOpMap.find(Key); LatestOp != m_LatestOpMap.end())
+	{
+		return LatestOp->second;
+	}
+	return -1;
+}
+
 std::optional<CbObject>
 ProjectStore::Oplog::GetOpByKey(const Oid& Key)
 {
@@ -2240,14 +2501,17 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, AuthMgr& AuthMgr)
+HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
 : m_Log(logging::Get("project"))
 , m_CidStore(Store)
 , m_ProjectStore(Projects)
+, m_StatsService(StatsService)
 , m_AuthMgr(AuthMgr)
 {
 	using namespace std::literals;
 
+	m_StatsService.RegisterHandler("prj", *this);
+
 	m_Router.AddPattern("project", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("log", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("op", "([[:digit:]]+?)");
@@ -3174,10 +3438,231 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 			m_ProjectStore->Rpc(HttpReq, ProjectId, OplogId, std::move(Payload), m_AuthMgr);
 		},
 		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"details\\$",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq = Req.ServerRequest();
+
+			HttpServerRequest::QueryParams Params			 = HttpReq.GetQueryParams();
+			bool						   CSV				 = Params.GetValue("csv") == "true";
+			bool						   Details			 = Params.GetValue("details") == "true";
+			bool						   OpDetails		 = Params.GetValue("opdetails") == "true";
+			bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+			if (CSV)
+			{
+				ExtendableStringBuilder<4096> CSVWriter;
+				CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+				m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+					Project.IterateOplogs([&](ProjectStore::Oplog& Oplog) {
+						Oplog.IterateOplogWithKey(
+							[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
+								CSVWriteOp(m_CidStore,
+										   Project.Identifier,
+										   Oplog.OplogId(),
+										   Details,
+										   AttachmentDetails,
+										   LSN,
+										   Key,
+										   Op,
+										   CSVWriter);
+							});
+					});
+				});
+
+				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+			}
+			else
+			{
+				CbObjectWriter Cbo;
+				Cbo.BeginArray("projects");
+				{
+					m_ProjectStore->DiscoverProjects();
+
+					m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
+						std::vector<std::string> OpLogs = Project.ScanForOplogs();
+						CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+					});
+				}
+				Cbo.EndArray();
+				HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+			}
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"details\\$/{project}",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq	 = Req.ServerRequest();
+			const auto&		   ProjectId = Req.GetCapture(1);
+
+			HttpServerRequest::QueryParams Params			 = HttpReq.GetQueryParams();
+			bool						   CSV				 = Params.GetValue("csv") == "true";
+			bool						   Details			 = Params.GetValue("details") == "true";
+			bool						   OpDetails		 = Params.GetValue("opdetails") == "true";
+			bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+			Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
+			if (!FoundProject)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+			ProjectStore::Project& Project = *FoundProject.Get();
+			if (CSV)
+			{
+				ExtendableStringBuilder<4096> CSVWriter;
+				CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+				FoundProject->IterateOplogs([&](ProjectStore::Oplog& Oplog) {
+					Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int		  LSN,
+																											   const Oid& Key,
+																											   CbObject	  Op) {
+						CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+					});
+				});
+				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+			}
+			else
+			{
+				CbObjectWriter			 Cbo;
+				std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
+				Cbo.BeginArray("projects");
+				{
+					CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+				}
+				Cbo.EndArray();
+				HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+			}
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"details\\$/{project}/{log}",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq	 = Req.ServerRequest();
+			const auto&		   ProjectId = Req.GetCapture(1);
+			const auto&		   OplogId	 = Req.GetCapture(2);
+
+			HttpServerRequest::QueryParams Params			 = HttpReq.GetQueryParams();
+			bool						   CSV				 = Params.GetValue("csv") == "true";
+			bool						   Details			 = Params.GetValue("details") == "true";
+			bool						   OpDetails		 = Params.GetValue("opdetails") == "true";
+			bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+			Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
+			if (!FoundProject)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+			ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
+
+			if (!FoundLog)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			ProjectStore::Project& Project = *FoundProject.Get();
+			ProjectStore::Oplog&   Oplog   = *FoundLog;
+			if (CSV)
+			{
+				ExtendableStringBuilder<4096> CSVWriter;
+				CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+				Oplog.IterateOplogWithKey(
+					[this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObject Op) {
+						CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+					});
+				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+			}
+			else
+			{
+				CbObjectWriter Cbo;
+				Cbo.BeginArray("oplogs");
+				{
+					CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+				}
+				Cbo.EndArray();
+				HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+			}
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"details\\$/{project}/{log}/{chunk}",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq	 = Req.ServerRequest();
+			const auto&		   ProjectId = Req.GetCapture(1);
+			const auto&		   OplogId	 = Req.GetCapture(2);
+			const auto&		   ChunkId	 = Req.GetCapture(3);
+
+			HttpServerRequest::QueryParams Params			 = HttpReq.GetQueryParams();
+			bool						   CSV				 = Params.GetValue("csv") == "true";
+			bool						   Details			 = Params.GetValue("details") == "true";
+			bool						   OpDetails		 = Params.GetValue("opdetails") == "true";
+			bool						   AttachmentDetails = Params.GetValue("attachmentdetails") == "true";
+
+			Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId);
+			if (!FoundProject)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+			ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId);
+
+			if (!FoundLog)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			if (ChunkId.size() != 2 * sizeof(Oid::OidBits))
+			{
+				return HttpReq.WriteResponse(
+					HttpResponseCode::BadRequest,
+					HttpContentType::kText,
+					fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId));
+			}
+
+			const Oid			   ObjId   = Oid::FromHexString(ChunkId);
+			ProjectStore::Project& Project = *FoundProject.Get();
+			ProjectStore::Oplog&   Oplog   = *FoundLog;
+
+			int LSN = Oplog.GetOpIndexByKey(ObjId);
+			if (LSN == -1)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+			std::optional<CbObject> Op = Oplog.GetOpByIndex(LSN);
+			if (!Op.has_value())
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			if (CSV)
+			{
+				ExtendableStringBuilder<4096> CSVWriter;
+				CSVHeader(Details, AttachmentDetails, CSVWriter);
+
+				CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter);
+				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
+			}
+			else
+			{
+				CbObjectWriter Cbo;
+				Cbo.BeginArray("ops");
+				{
+					CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo);
+				}
+				Cbo.EndArray();
+				HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+			}
+		},
+		HttpVerb::kGet);
 }
 
 HttpProjectService::~HttpProjectService()
 {
+	m_StatsService.UnregisterHandler("prj", *this);
 }
 
 const char*
@@ -3195,6 +3680,40 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
 	}
 }
 
+void
+HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+	const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
+	const CidStoreSize	CidSize	  = m_CidStore.TotalSize();
+
+	CbObjectWriter Cbo;
+	Cbo.BeginObject("store");
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "disk" << StoreSize.DiskSize;
+			Cbo << "memory" << StoreSize.MemorySize;
+		}
+		Cbo.EndObject();
+	}
+	Cbo.EndObject();
+
+	Cbo.BeginObject("cid");
+	{
+		Cbo.BeginObject("size");
+		{
+			Cbo << "tiny" << CidSize.TinySize;
+			Cbo << "small" << CidSize.SmallSize;
+			Cbo << "large" << CidSize.LargeSize;
+			Cbo << "total" << CidSize.TotalSize;
+		}
+		Cbo.EndObject();
+	}
+	Cbo.EndObject();
+
+	return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 #if ZEN_WITH_TESTS
diff --git a/zenserver/projectstore/projectstore.h b/zenserver/projectstore/projectstore.h
index 928a74f59..e4f664b85 100644
--- a/zenserver/projectstore/projectstore.h
+++ b/zenserver/projectstore/projectstore.h
@@ -7,6 +7,8 @@
 #include <zenhttp/httpserver.h>
 #include <zenstore/gc.h>
 
+#include "monitoring/httpstats.h"
+
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_map.h>
 ZEN_THIRD_PARTY_INCLUDES_END
@@ -80,8 +82,10 @@ public:
 
 		void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
 		void IterateOplog(std::function<void(CbObject)>&& Fn);
+		void IterateOplogWithKey(std::function<void(int, const Oid&, CbObject)>&& Fn);
 		std::optional<CbObject> GetOpByKey(const Oid& Key);
 		std::optional<CbObject> GetOpByIndex(int Index);
+		int						GetOpIndexByKey(const Oid& Key);
 
 		IoBuffer FindChunk(Oid ChunkId);
 
@@ -341,15 +345,17 @@ private:
 // refs:
 //
 
-class HttpProjectService : public HttpService
+class HttpProjectService : public HttpService, public IHttpStatsProvider
 {
 public:
-	HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, AuthMgr& AuthMgr);
+	HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr);
 	~HttpProjectService();
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
+	virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+
 private:
 	inline spdlog::logger& Log() { return m_Log; }
 
@@ -357,6 +363,7 @@ private:
 	CidStore&		  m_CidStore;
 	HttpRequestRouter m_Router;
 	Ref<ProjectStore> m_ProjectStore;
+	HttpStatsService& m_StatsService;
 	AuthMgr&		  m_AuthMgr;
 };
 
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index e3b364ea1..749f762f0 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -263,7 +263,7 @@ public:
 		ZEN_INFO("instantiating project service");
 
 		m_ProjectStore = new zen::ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager);
-		m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore, *m_AuthMgr});
+		m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr});
 
 #if ZEN_WITH_COMPUTE_SERVICES
 		if (ServerOptions.ComputeServiceEnabled)