oplog and cache stats (#244)

* basic oplog stats
* add GetValueStats to cache store
* RwLock::ExclusiveLockScope -> RwLock::SharedLockScope
* add rawhash and attachment count to CacheValueStats
* added cache-stats and project-stats commands
* add cast to make Mac overload detection happy
* fix accept type in cache-stats command
* Add options to project-stats command
* use resource paths for stats in project store
* use resource paths for stats in cache
* fix cache-info and project-info url discriminator
* more control over details$ output
* cleanup
* changelog

1 files changed, 95 insertions, 5 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 55af85ade..44574ae19 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -421,6 +421,12 @@ ZenCacheNamespace::GetBucketInfo(std::string_view Bucket) const
 	return Info;
 }
 
+CacheValueDetails::NamespaceDetails
+ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
+{
+	return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter);
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 ZenCacheMemoryLayer::ZenCacheMemoryLayer()
@@ -589,7 +595,7 @@ ZenCacheMemoryLayer::GetInfo() const
 std::optional<ZenCacheMemoryLayer::BucketInfo>
 ZenCacheMemoryLayer::GetBucketInfo(std::string_view Bucket) const
 {
-	RwLock::ExclusiveLockScope _(m_Lock);
+	RwLock::SharedLockScope _(m_Lock);
 
 	if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end())
 	{
@@ -1195,7 +1201,7 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const bool IsNew)
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey)
+ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const
 {
 	char HexString[sizeof(HashKey.Hash) * 2];
 	ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, HexString);
@@ -1212,7 +1218,7 @@ ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& H
 }
 
 IoBuffer
-ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc)
+ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc) const
 {
 	BlockStoreLocation Location = Loc.GetBlockLocation(m_PayloadAlignment);
 
@@ -1226,7 +1232,7 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc)
 }
 
 IoBuffer
-ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey)
+ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const
 {
 	ExtendablePathBuilder<256> DataFilePath;
 	BuildPath(DataFilePath, HashKey);
@@ -2036,6 +2042,50 @@ ZenCacheDiskLayer::CacheBucket::EntryCount() const
 	return static_cast<uint64_t>(m_Index.size());
 }
 
+CacheValueDetails::ValueDetails
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, size_t Index) const
+{
+	std::vector<IoHash>	 Attachments;
+	const BucketPayload& Payload = m_Payloads[Index];
+	if (Payload.Location.IsFlagSet(DiskLocation::kStructured))
+	{
+		IoBuffer Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile) ? GetStandaloneCacheValue(Payload.Location, Key)
+																				   : GetInlineCacheValue(Payload.Location);
+		CbObject Obj(SharedBuffer{Value});
+		Obj.IterateAttachments([&Attachments](CbFieldView Field) { Attachments.emplace_back(Field.AsAttachment()); });
+	}
+	return CacheValueDetails::ValueDetails{.Size		= Payload.Location.Size(),
+										   .RawSize		= Payload.RawSize,
+										   .RawHash		= Payload.RawHash,
+										   .LastAccess	= m_AccessTimes[Index],
+										   .Attachments = std::move(Attachments),
+										   .ContentType = Payload.Location.GetContentType()};
+}
+
+CacheValueDetails::BucketDetails
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilter) const
+{
+	CacheValueDetails::BucketDetails Details;
+	RwLock::SharedLockScope			 _(m_IndexLock);
+	if (ValueFilter.empty())
+	{
+		Details.Values.reserve(m_Index.size());
+		for (const auto& It : m_Index)
+		{
+			Details.Values.insert_or_assign(It.first, GetValueDetails(It.first, It.second));
+		}
+	}
+	else
+	{
+		IoHash Key = IoHash::FromHexString(ValueFilter);
+		if (auto It = m_Index.find(Key); It != m_Index.end())
+		{
+			Details.Values.insert_or_assign(It->first, GetValueDetails(It->first, It->second));
+		}
+	}
+	return Details;
+}
+
 void
 ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx)
 {
@@ -2505,7 +2555,7 @@ ZenCacheDiskLayer::GetInfo() const
 std::optional<ZenCacheDiskLayer::BucketInfo>
 ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const
 {
-	RwLock::ExclusiveLockScope _(m_Lock);
+	RwLock::SharedLockScope _(m_Lock);
 
 	if (auto It = m_Buckets.find(std::string(Bucket)); It != m_Buckets.end())
 	{
@@ -2514,6 +2564,26 @@ ZenCacheDiskLayer::GetBucketInfo(std::string_view Bucket) const
 	return {};
 }
 
+CacheValueDetails::NamespaceDetails
+ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
+{
+	RwLock::SharedLockScope				_(m_Lock);
+	CacheValueDetails::NamespaceDetails Details;
+	if (BucketFilter.empty())
+	{
+		Details.Buckets.reserve(BucketFilter.empty() ? m_Buckets.size() : 1);
+		for (auto& Kv : m_Buckets)
+		{
+			Details.Buckets[Kv.first] = Kv.second->GetValueDetails(ValueFilter);
+		}
+	}
+	else if (auto It = m_Buckets.find(std::string(BucketFilter)); It != m_Buckets.end())
+	{
+		Details.Buckets[It->first] = It->second->GetValueDetails(ValueFilter);
+	}
+	return Details;
+}
+
 //////////////////////////// ZenCacheStore
 
 static constexpr std::string_view UE4DDCNamespaceName = "ue4.ddc";
@@ -2568,6 +2638,7 @@ ZenCacheStore::Get(std::string_view Namespace, std::string_view Bucket, const Io
 		return Store->Get(Bucket, HashKey, OutValue);
 	}
 	ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get, bucket '{}', key '{}'", Namespace, Bucket, HashKey.ToHexString());
+
 	return false;
 }
 
@@ -2619,6 +2690,25 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Scrub(Ctx); });
 }
 
+CacheValueDetails
+ZenCacheStore::GetValueDetails(const std::string_view NamespaceFilter,
+							   const std::string_view BucketFilter,
+							   const std::string_view ValueFilter) const
+{
+	CacheValueDetails Details;
+	if (NamespaceFilter.empty())
+	{
+		IterateNamespaces([&](std::string_view Namespace, ZenCacheNamespace& Store) {
+			Details.Namespaces[std::string(Namespace)] = Store.GetValueDetails(BucketFilter, ValueFilter);
+		});
+	}
+	else if (const ZenCacheNamespace* Store = FindNamespace(NamespaceFilter); Store != nullptr)
+	{
+		Details.Namespaces[std::string(NamespaceFilter)] = Store->GetValueDetails(BucketFilter, ValueFilter);
+	}
+	return Details;
+}
+
 ZenCacheNamespace*
 ZenCacheStore::GetNamespace(std::string_view Namespace)
 {