Add info (GET) endpoints for structured cache (#211)

* Add GET requests on cache/namespace/bucket level
* Add root route for project store requests (same as /list)
* Add markerpath to oplog info object
* Add totalsize, opcount and expired to oplog info
* Changelog

1 files changed, 98 insertions, 21 deletions

diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index e6b849c0a..e6e9942bb 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -143,6 +143,26 @@ static_assert(sizeof(DiskIndexEntry) == 32);
 class ZenCacheMemoryLayer
 {
 public:
+	struct Configuration
+	{
+		uint64_t TargetFootprintBytes = 16 * 1024 * 1024;
+		uint64_t ScavengeThreshold	  = 4 * 1024 * 1024;
+	};
+
+	struct BucketInfo
+	{
+		uint64_t EntryCount = 0;
+		uint64_t TotalSize	= 0;
+	};
+
+	struct Info
+	{
+		Configuration			 Config;
+		std::vector<std::string> BucketNames;
+		uint64_t				 EntryCount = 0;
+		uint64_t				 TotalSize	= 0;
+	};
+
 	ZenCacheMemoryLayer();
 	~ZenCacheMemoryLayer();
 
@@ -155,11 +175,8 @@ public:
 	void	 Reset();
 	uint64_t TotalSize() const;
 
-	struct Configuration
-	{
-		uint64_t TargetFootprintBytes = 16 * 1024 * 1024;
-		uint64_t ScavengeThreshold	  = 4 * 1024 * 1024;
-	};
+	Info					  GetInfo() const;
+	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
 
 	const Configuration& GetConfiguration() const { return m_Configuration; }
 	void				 SetConfiguration(const Configuration& NewConfig) { m_Configuration = NewConfig; }
@@ -186,7 +203,7 @@ private:
 			}
 		};
 
-		RwLock								m_BucketLock;
+		mutable RwLock						m_BucketLock;
 		tsl::robin_map<IoHash, BucketValue> m_CacheMap;
 		std::atomic_uint64_t				m_TotalSize{};
 
@@ -196,6 +213,7 @@ private:
 		void			Scrub(ScrubContext& Ctx);
 		void			GatherAccessTimes(std::vector<zen::access_tracking::KeyAccessTime>& AccessTimes);
 		inline uint64_t TotalSize() const { return m_TotalSize; }
+		uint64_t		EntryCount() const;
 	};
 
 	mutable RwLock												  m_Lock;
@@ -210,6 +228,25 @@ private:
 class ZenCacheDiskLayer
 {
 public:
+	struct Configuration
+	{
+		std::filesystem::path RootDir;
+	};
+
+	struct BucketInfo
+	{
+		uint64_t EntryCount = 0;
+		uint64_t TotalSize	= 0;
+	};
+
+	struct Info
+	{
+		Configuration			 Config;
+		std::vector<std::string> BucketNames;
+		uint64_t				 EntryCount = 0;
+		uint64_t				 TotalSize	= 0;
+	};
+
 	explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir);
 	~ZenCacheDiskLayer();
 
@@ -222,10 +259,14 @@ public:
 	void GatherReferences(GcContext& GcCtx);
 	void CollectGarbage(GcContext& GcCtx);
 	void UpdateAccessTimes(const zen::access_tracking::AccessTimes& AccessTimes);
+	// void IterateBuckets(const std::function<void(std::string_view Bucket)>& Callback) const;
 
 	void	 DiscoverBuckets();
 	uint64_t TotalSize() const;
 
+	Info					  GetInfo() const;
+	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
+
 private:
 	/** A cache bucket manages a single directory containing
 		metadata and data for that bucket
@@ -246,6 +287,7 @@ private:
 		void UpdateAccessTimes(const std::vector<zen::access_tracking::KeyAccessTime>& AccessTimes);
 
 		inline uint64_t TotalSize() const { return m_TotalStandaloneSize.load(std::memory_order::relaxed) + m_BlockStore.TotalSize(); }
+		uint64_t		EntryCount() const;
 
 	private:
 		const uint64_t MaxBlockSize		  = 1ull << 30;
@@ -286,8 +328,8 @@ private:
 
 		using IndexMap = tsl::robin_map<IoHash, IndexEntry, IoHash::Hasher>;
 
-		RwLock	 m_IndexLock;
-		IndexMap m_Index;
+		mutable RwLock m_IndexLock;
+		IndexMap	   m_Index;
 
 		std::atomic_uint64_t m_TotalStandaloneSize{};
 
@@ -325,19 +367,39 @@ private:
 class ZenCacheNamespace final : public RefCounted, public GcStorage, public GcContributor
 {
 public:
+	struct Configuration
+	{
+		std::filesystem::path RootDir;
+		uint64_t			  DiskLayerThreshold = 0;
+	};
+	struct BucketInfo
+	{
+		ZenCacheDiskLayer::BucketInfo	DiskLayerInfo;
+		ZenCacheMemoryLayer::BucketInfo MemoryLayerInfo;
+	};
+	struct Info
+	{
+		Configuration			  Config;
+		std::vector<std::string>  BucketNames;
+		ZenCacheDiskLayer::Info	  DiskLayerInfo;
+		ZenCacheMemoryLayer::Info MemoryLayerInfo;
+	};
+
 	ZenCacheNamespace(GcManager& Gc, const std::filesystem::path& RootDir);
 	~ZenCacheNamespace();
 
-	bool				  Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
-	void				  Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
-	bool				  Drop();
-	bool				  DropBucket(std::string_view Bucket);
-	void				  Flush();
-	void				  Scrub(ScrubContext& Ctx);
-	uint64_t			  DiskLayerThreshold() const { return m_DiskLayerSizeThreshold; }
-	virtual void		  GatherReferences(GcContext& GcCtx) override;
-	virtual void		  CollectGarbage(GcContext& GcCtx) override;
-	virtual GcStorageSize StorageSize() const override;
+	bool					  Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+	void					  Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+	bool					  Drop();
+	bool					  DropBucket(std::string_view Bucket);
+	void					  Flush();
+	void					  Scrub(ScrubContext& Ctx);
+	uint64_t				  DiskLayerThreshold() const { return m_DiskLayerSizeThreshold; }
+	virtual void			  GatherReferences(GcContext& GcCtx) override;
+	virtual void			  CollectGarbage(GcContext& GcCtx) override;
+	virtual GcStorageSize	  StorageSize() const override;
+	Info					  GetInfo() const;
+	std::optional<BucketInfo> GetBucketInfo(std::string_view Bucket) const;
 
 private:
 	std::filesystem::path m_RootDir;
@@ -364,7 +426,16 @@ public:
 	struct Configuration
 	{
 		std::filesystem::path BasePath;
-		bool				  AllowAutomaticCreationOfNamespaces = true;
+		bool				  AllowAutomaticCreationOfNamespaces = false;
+	};
+
+	struct Info
+	{
+		Configuration			 Config;
+		std::vector<std::string> NamespaceNames;
+		uint64_t				 DiskEntryCount	  = 0;
+		uint64_t				 MemoryEntryCount = 0;
+		GcStorageSize			 StorageSize;
 	};
 
 	ZenCacheStore(GcManager& Gc, const Configuration& Configuration);
@@ -378,10 +449,16 @@ public:
 	void Scrub(ScrubContext& Ctx);
 
 	GcStorageSize StorageSize() const;
+	//	const Configuration& GetConfiguration() const { return m_Configuration; }
+
+	Info										 GetInfo() const;
+	std::optional<ZenCacheNamespace::Info>		 GetNamespaceInfo(std::string_view Namespace);
+	std::optional<ZenCacheNamespace::BucketInfo> GetBucketInfo(std::string_view Namespace, std::string_view Bucket);
 
 private:
-	ZenCacheNamespace* GetNamespace(std::string_view Namespace);
-	void			   IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const;
+	const ZenCacheNamespace* FindNamespace(std::string_view Namespace) const;
+	ZenCacheNamespace*		 GetNamespace(std::string_view Namespace);
+	void IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const;
 
 	typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NamespaceMap;