From efd016d84d0940bf616e3efef135532cbf5fedef Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Mon, 21 Oct 2024 15:40:13 +0200
Subject: bucket size queries (#203)

- Feature: Added options --bucketsize and --bucketsizes to zen cache-info to get data sizes in cache buckets and attachments
---
 src/zenstore/include/zenstore/cache/cachedisklayer.h   | 18 +++++++++++++++++-
 src/zenstore/include/zenstore/cache/cacheshared.h      |  8 ++++++++
 .../include/zenstore/cache/structuredcachestore.h      |  4 ++++
 3 files changed, 29 insertions(+), 1 deletion(-)

(limited to 'src/zenstore/include')

diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index f8ce8641c..4b7cf6101 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -205,6 +205,8 @@ public:
 	void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time);
 #endif	// ZEN_WITH_TESTS
 
+	bool GetContentStats(std::string_view BucketName, CacheContentStats& OutContentStats) const;
+
 	/** A cache bucket manages a single directory containing
 		metadata and data for that bucket
 	  */
@@ -230,7 +232,21 @@ public:
 		void	 Flush();
 		void	 ScrubStorage(ScrubContext& Ctx);
 		RwLock::SharedLockScope GetGcReferencerLock();
-		bool					GetReferences(GcCtx& Ctx, bool StateIsAlreadyLocked, std::vector<IoHash>& OutReferences);
+
+		struct ReferencesStats
+		{
+			std::vector<uint64_t> ValueSizes;
+			uint64_t			  StructuredValuesCount = 0;
+			uint64_t			  StandaloneValuesCount = 0;
+		};
+
+		bool GetReferences(const LoggerRef&		Logger,
+						   std::atomic_bool&	IsCancelledFlag,
+						   bool					StateIsAlreadyLocked,
+						   bool					ReadCacheAttachmentMetaData,
+						   bool					WriteCacheAttachmentMetaData,
+						   std::vector<IoHash>& OutReferences,
+						   ReferencesStats*		OptionalOutReferencesStats);
 
 		bool ReadAttachmentsFromMetaData(uint32_t					  BlockIndex,
 										 std::span<const IoHash>	  InlineKeys,
diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h
index 2d5b9cbc3..9b45c7b21 100644
--- a/src/zenstore/include/zenstore/cache/cacheshared.h
+++ b/src/zenstore/include/zenstore/cache/cacheshared.h
@@ -57,6 +57,14 @@ struct CacheValueDetails
 	std::unordered_map<std::string, NamespaceDetails> Namespaces;
 };
 
+struct CacheContentStats
+{
+	std::vector<uint64_t> ValueSizes;
+	uint64_t			  StructuredValuesCount = 0;
+	uint64_t			  StandaloneValuesCount = 0;
+	std::vector<IoHash>	  Attachments;
+};
+
 bool IsKnownBadBucketName(std::string_view BucketName);
 bool ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer);
 
diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h
index 50e40042a..dcdca71c6 100644
--- a/src/zenstore/include/zenstore/cache/structuredcachestore.h
+++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h
@@ -120,6 +120,8 @@ public:
 	void EnableUpdateCapture();
 	void DisableUpdateCapture();
 
+	bool GetContentStats(std::string_view BucketName, CacheContentStats& OutContentStats) const;
+
 #if ZEN_WITH_TESTS
 	void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time);
 #endif	// ZEN_WITH_TESTS
@@ -285,6 +287,8 @@ public:
 	void					 DisableUpdateCapture();
 	std::vector<std::string> GetCapturedNamespaces();
 
+	bool GetContentStats(std::string_view Namespace, std::string_view BucketName, CacheContentStats& OutContentStats) const;
+
 private:
 	const ZenCacheNamespace* FindNamespace(std::string_view Namespace) const;
 	ZenCacheNamespace*		 GetNamespace(std::string_view Namespace);
-- 
cgit v1.2.3