cas: Hooked up GC to structured cache

8 files changed, 115 insertions, 31 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 15f7b5a7a..eb3c1d47a 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -6,6 +6,7 @@
 #include <zencore/windows.h>
 
 #include <zencore/compactbinary.h>
+#include <zencore/compactbinaryvalidation.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
@@ -19,6 +20,7 @@
 
 #include <concepts>
 #include <filesystem>
+#include <memory_resource>
 #include <unordered_map>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -32,7 +34,7 @@ namespace zen {
 
 using namespace fmt::literals;
 
-ZenCacheStore::ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir) : m_DiskLayer{Cas, RootDir}
+ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) : GcContributor(Gc), m_DiskLayer{RootDir}
 {
 	ZEN_INFO("initializing structured cache at '{}'", RootDir);
 	CreateDirectories(RootDir);
@@ -81,6 +83,25 @@ ZenCacheStore::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCa
 
 	m_DiskLayer.Put(InBucket, HashKey, Value);
 
+#if ZEN_REF_TRACKING
+	if (Value.Value.GetContentType() == ZenContentType::kCbObject)
+	{
+		if (ValidateCompactBinary(Value.Value, CbValidateMode::All) == CbValidateError::None)
+		{
+			CbObject Object{SharedBuffer(Value.Value)};
+
+			uint8_t								TempBuffer[8 * sizeof(IoHash)];
+			std::pmr::monotonic_buffer_resource Linear{TempBuffer, sizeof TempBuffer};
+			std::pmr::polymorphic_allocator		Allocator{&Linear};
+			std::pmr::vector<IoHash>			CidReferences{Allocator};
+
+			Object.IterateAttachments([&](CbFieldView Field) { CidReferences.push_back(Field.AsAttachment()); });
+
+			m_Gc.OnNewCidReferences(CidReferences);
+		}
+	}
+#endif
+
 	if (Value.Value.Size() <= m_DiskLayerSizeThreshold)
 	{
 		m_MemLayer.Put(InBucket, HashKey, Value);
@@ -126,7 +147,8 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 void
 ZenCacheStore::GarbageCollect(GcContext& GcCtx)
 {
-	ZEN_UNUSED(GcCtx);
+	m_MemLayer.GarbageCollect(GcCtx);
+	m_DiskLayer.GarbageCollect(GcCtx);
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -359,23 +381,21 @@ static_assert(sizeof(DiskIndexEntry) == 36);
 
 struct ZenCacheDiskLayer::CacheBucket
 {
-	CacheBucket(CasStore& Cas);
+	CacheBucket();
 	~CacheBucket();
 
 	void		OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
 	static bool Delete(std::filesystem::path BucketDir);
-
-	bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
-	void Put(const IoHash& HashKey, const ZenCacheValue& Value);
-	void Drop();
-	void Flush();
-	void Scrub(ScrubContext& Ctx);
-	void GarbageCollect(GcContext& GcCtx);
+	bool		Get(const IoHash& HashKey, ZenCacheValue& OutValue);
+	void		Put(const IoHash& HashKey, const ZenCacheValue& Value);
+	void		Drop();
+	void		Flush();
+	void		Scrub(ScrubContext& Ctx);
+	void		GarbageCollect(GcContext& GcCtx);
 
 	inline bool IsOk() const { return m_IsOk; }
 
 private:
-	CasStore&			  m_CasStore;
 	std::filesystem::path m_BucketDir;
 	Oid					  m_BucketId;
 	bool				  m_IsOk				 = false;
@@ -406,7 +426,7 @@ private:
 	inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; }
 };
 
-ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas)
+ZenCacheDiskLayer::CacheBucket::CacheBucket()
 {
 }
 
@@ -840,7 +860,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 
 //////////////////////////////////////////////////////////////////////////
 
-ZenCacheDiskLayer::ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir) : m_RootDir(RootDir), m_CasStore(Cas)
+ZenCacheDiskLayer::ZenCacheDiskLayer(const std::filesystem::path& RootDir) : m_RootDir(RootDir)
 {
 }
 
@@ -874,7 +894,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 		}
 		else
 		{
-			auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
+			auto It = m_Buckets.try_emplace(std::string(InBucket));
 			Bucket	= &It.first->second;
 
 			std::filesystem::path BucketPath = m_RootDir;
@@ -917,7 +937,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 		}
 		else
 		{
-			auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
+			auto It = m_Buckets.try_emplace(std::string(InBucket));
 			Bucket	= &It.first->second;
 
 			std::filesystem::path bucketPath = m_RootDir;
@@ -973,7 +993,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 		}
 		else
 		{
-			auto InsertResult = m_Buckets.try_emplace(BucketName8, m_CasStore);
+			auto InsertResult = m_Buckets.try_emplace(BucketName8);
 
 			std::filesystem::path BucketPath = m_RootDir;
 			BucketPath /= BucketName8;
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 4753af627..042d2ab3d 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -8,6 +8,7 @@
 #include <zencore/thread.h>
 #include <zencore/uid.h>
 #include <zenstore/cas.h>
+#include <zenstore/gc.h>
 
 #pragma warning(push)
 #pragma warning(disable : 4127)
@@ -22,6 +23,7 @@ namespace zen {
 
 class WideStringBuilderBase;
 class CasStore;
+class CasGc;
 
 /******************************************************************************
 
@@ -50,6 +52,9 @@ struct ZenCacheValue
 
 	Intended for small values which are frequently accessed
 
+	This should have a better memory management policy to maintain reasonable
+	footprint.
+
  */
 class ZenCacheMemoryLayer
 {
@@ -101,7 +106,7 @@ private:
 class ZenCacheDiskLayer
 {
 public:
-	ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir);
+	ZenCacheDiskLayer(const std::filesystem::path& RootDir);
 	~ZenCacheDiskLayer();
 
 	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -119,30 +124,29 @@ private:
 	  */
 	struct CacheBucket;
 
-	CasStore&									 m_CasStore;
 	std::filesystem::path						 m_RootDir;
 	RwLock										 m_Lock;
 	std::unordered_map<std::string, CacheBucket> m_Buckets;	 // TODO: make this case insensitive
 };
 
-class ZenCacheStore
+class ZenCacheStore : public GcContributor
 {
 public:
-	ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir);
+	ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir);
 	~ZenCacheStore();
 
-	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
-	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
-	bool DropBucket(std::string_view Bucket);
-	void Flush();
-	void Scrub(ScrubContext& Ctx);
-	void GarbageCollect(GcContext& GcCtx);
+	bool		 Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
+	void		 Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+	bool		 DropBucket(std::string_view Bucket);
+	void		 Flush();
+	void		 Scrub(ScrubContext& Ctx);
+	virtual void GarbageCollect(GcContext& GcCtx) override;
 
 private:
 	std::filesystem::path m_RootDir;
 	ZenCacheMemoryLayer	  m_MemLayer;
 	ZenCacheDiskLayer	  m_DiskLayer;
-	uint64_t			  m_DiskLayerSizeThreshold = 4 * 1024;
+	uint64_t			  m_DiskLayerSizeThreshold = 1 * 1024;
 	uint64_t			  m_LastScrubTime		   = 0;
 };
 
diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp
index 807bba3b3..589637b62 100644
--- a/zenstore/CAS.cpp
+++ b/zenstore/CAS.cpp
@@ -100,6 +100,7 @@ public:
 	virtual void				   FilterChunks(CasChunkSet& InOutChunks) override;
 	virtual void				   Flush() override;
 	virtual void				   Scrub(ScrubContext& Ctx) override;
+	virtual void				   GarbageCollect(GcContext& GcCtx) override;
 
 private:
 	CasContainerStrategy m_TinyStrategy;
@@ -234,6 +235,12 @@ CasImpl::Scrub(ScrubContext& Ctx)
 	m_LargeStrategy.Scrub(Ctx);
 }
 
+void 
+CasImpl::GarbageCollect(GcContext& GcCtx)
+{
+	m_LargeStrategy.GarbageCollect(GcCtx);
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 CasStore*
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 612f87c7c..ee027b261 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -258,6 +258,11 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
 }
 
 void
+CasContainerStrategy::GarbageCollect(GcContext& GcCtx)
+{
+}
+
+void
 CasContainerStrategy::MakeSnapshot()
 {
 	RwLock::SharedLockScope _(m_LocationMapLock);
diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h
index a512c3d93..8f7c0213b 100644
--- a/zenstore/compactcas.h
+++ b/zenstore/compactcas.h
@@ -61,6 +61,7 @@ struct CasContainerStrategy
 	void				   Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore);
 	void				   Flush();
 	void				   Scrub(ScrubContext& Ctx);
+	void				   GarbageCollect(GcContext& GcCtx);
 
 private:
 	const CasStoreConfiguration&   m_Config;
diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
index e7c8f3a1a..c25ea344c 100644
--- a/zenstore/gc.cpp
+++ b/zenstore/gc.cpp
@@ -35,6 +35,18 @@ GcContext::ContributeCas(std::span<const IoHash> Cas)
 
 //////////////////////////////////////////////////////////////////////////
 
+GcContributor::GcContributor(CasGc& Gc) : m_Gc(Gc)
+{
+	m_Gc.AddGcContributor(this);
+}
+
+GcContributor::~GcContributor()
+{
+	m_Gc.RemoveGcContributor(this);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
 CasGc::CasGc(CasStore& Store) : m_CasStore(Store)
 {
 }
@@ -43,13 +55,27 @@ CasGc::~CasGc()
 {
 }
 
+void 
+CasGc::AddGcContributor(GcContributor* Contributor)
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+	m_GcContribs.push_back(Contributor);
+}
+
+void 
+CasGc::RemoveGcContributor(GcContributor* Contributor)
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+	std::erase_if(m_GcContribs, [&](GcContributor* $) { return $ == Contributor; });
+}
+
 void
 CasGc::CollectGarbage()
 {
 }
 
 void
-CasGc::OnNewReferences(std::span<IoHash> Hashes)
+CasGc::OnNewCidReferences(std::span<IoHash> Hashes)
 {
 	ZEN_UNUSED(Hashes);
 }
diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h
index dc7e260ab..738103f73 100644
--- a/zenstore/include/zenstore/CAS.h
+++ b/zenstore/include/zenstore/CAS.h
@@ -100,6 +100,7 @@ public:
 	virtual void		 FilterChunks(CasChunkSet& InOutChunks)				 = 0;
 	virtual void		 Flush()											 = 0;
 	virtual void		 Scrub(ScrubContext& Ctx)							 = 0;
+	virtual void		 GarbageCollect(GcContext& GcCtx)					 = 0;
 
 protected:
 	CasStoreConfiguration m_Config;
diff --git a/zenstore/include/zenstore/gc.h b/zenstore/include/zenstore/gc.h
index 33a43f4d2..dda33c9eb 100644
--- a/zenstore/include/zenstore/gc.h
+++ b/zenstore/include/zenstore/gc.h
@@ -3,12 +3,14 @@
 #pragma once
 
 #include <zencore/iohash.h>
+#include <zencore/thread.h>
 
 #include <span>
 
 namespace zen {
 
 class CasStore;
+class CasGc;
 struct IoHash;
 
 /** Garbage Collection context object
@@ -29,8 +31,20 @@ private:
 	std::unique_ptr<GcState> m_State;
 };
 
+class GcContributor
+{
+public:
+	GcContributor(CasGc& Gc);
+	~GcContributor();
+
+	virtual void GarbageCollect(GcContext& GcCtx) = 0;
+
+protected:
+	CasGc& m_Gc;
+};
+
 /** GC
-  */
+ */
 
 class CasGc
 {
@@ -38,12 +52,18 @@ public:
 	CasGc(CasStore& Store);
 	~CasGc();
 
+	void AddGcContributor(GcContributor* Contributor);
+	void RemoveGcContributor(GcContributor* Contributor);
+
 	void CollectGarbage();
 
-	void OnNewReferences(std::span<IoHash> Hashes);
+	void OnNewCidReferences(std::span<IoHash> Hashes);
 
 private:
-	CasStore& m_CasStore;
+	CasStore&				   m_CasStore;
+	RwLock					   m_Lock;
+	std::vector<GcContributor*> m_GcContribs;
 };
 
 }  // namespace zen
+