oplog and cache stats (#244)

* basic oplog stats
* add GetValueStats to cache store
* RwLock::ExclusiveLockScope -> RwLock::SharedLockScope
* add rawhash and attachment count to CacheValueStats
* added cache-stats and project-stats commands
* add cast to make Mac overload detection happy
* fix accept type in cache-stats command
* Add options to project-stats command
* use resource paths for stats in project store
* use resource paths for stats in cache
* fix cache-info and project-info url discriminator
* more control over details$ output
* cleanup
* changelog

1 files changed, 49 insertions, 14 deletions

diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index fe83f3c97..2ef4c4dcb 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -67,6 +67,31 @@ struct ZenCacheValue
 	IoHash	 RawHash = IoHash::Zero;
 };
 
+struct CacheValueDetails
+{
+	struct ValueDetails
+	{
+		uint64_t			Size;
+		uint64_t			RawSize;
+		IoHash				RawHash;
+		GcClock::Tick		LastAccess{};
+		std::vector<IoHash> Attachments;
+		ZenContentType		ContentType;
+	};
+
+	struct BucketDetails
+	{
+		std::unordered_map<IoHash, ValueDetails, IoHash::Hasher> Values;
+	};
+
+	struct NamespaceDetails
+	{
+		std::unordered_map<std::string, BucketDetails> Buckets;
+	};
+
+	std::unordered_map<std::string, NamespaceDetails> Namespaces;
+};
+
 //////////////////////////////////////////////////////////////////////////
 
 #pragma pack(push)
@@ -301,6 +326,8 @@ public:
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
 
+	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
+
 private:
 	/** A cache bucket manages a single directory containing
 		metadata and data for that bucket
@@ -323,6 +350,8 @@ private:
 		inline uint64_t TotalSize() const { return m_TotalStandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(); }
 		uint64_t		EntryCount() const;
 
+		CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const;
+
 	private:
 		const uint64_t MaxBlockSize		  = 1ull << 30;
 		uint64_t	   m_PayloadAlignment = 1ull << 4;
@@ -360,18 +389,18 @@ private:
 
 		std::atomic_uint64_t m_TotalStandaloneSize{};
 
-		void	 BuildPath(PathBuilderBase& Path, const IoHash& HashKey);
-		void	 PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
-		IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey);
-		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
-		IoBuffer GetInlineCacheValue(const DiskLocation& Loc);
-		void	 MakeIndexSnapshot();
-		uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
-		uint64_t ReadLogV2(const std::filesystem::path& LogPath, uint64_t LogPosition);
-		uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
-		void	 OpenLog(const bool IsNew);
-		void	 SaveManifest();
-
+		void							BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
+		void							PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+		IoBuffer						GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const;
+		void							PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+		IoBuffer						GetInlineCacheValue(const DiskLocation& Loc) const;
+		void							MakeIndexSnapshot();
+		uint64_t						ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
+		uint64_t						ReadLogV2(const std::filesystem::path& LogPath, uint64_t LogPosition);
+		uint64_t						ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
+		void							OpenLog(const bool IsNew);
+		void							SaveManifest();
+		CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, size_t Index) const;
 		// These locks are here to avoid contention on file creation, therefore it's sufficient
 		// that we take the same lock for the same hash
 		//
@@ -379,8 +408,8 @@ private:
 		// but we don't currently access them at particularly high frequency so it should not be
 		// an issue in practice
 
-		RwLock		   m_ShardedLocks[256];
-		inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; }
+		mutable RwLock m_ShardedLocks[256];
+		inline RwLock& LockForHash(const IoHash& Hash) const { return m_ShardedLocks[Hash.Hash[19]]; }
 	};
 
 	std::filesystem::path										  m_RootDir;
@@ -429,6 +458,8 @@ public:
 	Info					  GetInfo() const;
 	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
 
+	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
+
 private:
 	std::filesystem::path m_RootDir;
 	ZenCacheMemoryLayer	  m_MemLayer;
@@ -476,6 +507,10 @@ public:
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
 
+	CacheValueDetails GetValueDetails(const std::string_view NamespaceFilter,
+									  const std::string_view BucketFilter,
+									  const std::string_view ValueFilter) const;
+
 	GcStorageSize StorageSize() const;
 	//	const Configuration& GetConfiguration() const { return m_Configuration; }