New GC implementation (#459)

- Feature: New garbage collection implementation, still in evaluation mode. Enabled by `--gc-v2` command line option

1 files changed, 653 insertions, 1 deletions

diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 6fab14eee..516532528 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -64,7 +64,7 @@ ZenCacheNamespace::ZenCacheNamespace(GcManager& Gc, JobQueue& JobQueue, const st
 , m_JobQueue(JobQueue)
 , m_RootDir(RootDir)
 , m_Configuration(Config)
-, m_DiskLayer(m_JobQueue, m_RootDir, m_Configuration.DiskLayerConfig)
+, m_DiskLayer(m_Gc, m_JobQueue, m_RootDir, m_Configuration.DiskLayerConfig)
 {
 	ZEN_INFO("initializing structured cache at '{}'", m_RootDir);
 	CreateDirectories(m_RootDir);
@@ -232,6 +232,14 @@ ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const st
 	return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter);
 }
 
+#if ZEN_WITH_TESTS
+void
+ZenCacheNamespace::SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time)
+{
+	m_DiskLayer.SetAccessTime(Bucket, HashKey, Time);
+}
+#endif	// ZEN_WITH_TESTS
+
 //////////////////////////// ZenCacheStore
 
 ZEN_DEFINE_LOG_CATEGORY_STATIC(LogCacheActivity, "z$");
@@ -784,6 +792,73 @@ namespace testutils {
 		return Buf;
 	};
 
+	IoHash ToIoHash(const Oid& Id)
+	{
+		char OIdString[24 + 1];
+		Id.ToString(OIdString);
+		IoHash Key = IoHash::HashBuffer(OIdString, 24);
+		return Key;
+	}
+
+	std::pair<Oid, IoBuffer> CreateBinaryBlob(size_t Size)
+	{
+		uint64_t seed{Size};
+		auto	 next = [](uint64_t& seed) {
+			uint64_t z = (seed += UINT64_C(0x9E3779B97F4A7C15));
+			z		   = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4E5B9);
+			z		   = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB);
+			return z ^ (z >> 31);
+		};
+
+		IoBuffer  Data(Size);
+		uint64_t* DataPtr = reinterpret_cast<uint64_t*>(Data.MutableData());
+		while (Size > sizeof(uint64_t))
+		{
+			*DataPtr++ = next(seed);
+			Size -= sizeof(uint64_t);
+		}
+		uint64_t ByteNext	 = next(seed);
+		uint8_t* ByteDataPtr = reinterpret_cast<uint8_t*>(DataPtr);
+		while (Size > 0)
+		{
+			*ByteDataPtr++ = static_cast<uint8_t>(ByteNext & 0xff);
+			ByteNext >>= 8;
+			Size--;
+		}
+		return {Oid::NewOid(), Data};
+	}
+
+	std::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes)
+	{
+		std::vector<std::pair<Oid, CompressedBuffer>> Result;
+		Result.reserve(Sizes.size());
+		for (size_t Size : Sizes)
+		{
+			auto			 Blob		= CreateBinaryBlob(Size);
+			CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(Blob.second.Data(), Blob.second.Size()));
+			CHECK(!Store.ContainsChunk(Compressed.DecodeRawHash()));
+			Result.emplace_back(std::pair<Oid, CompressedBuffer>(Blob.first, Compressed));
+		}
+		return Result;
+	}
+
+	std::pair<IoHash, IoBuffer> CreateRecord(std::span<std::pair<Oid, CompressedBuffer>> Attachments)
+	{
+		Oid			   Id  = Oid::NewOid();
+		IoHash		   Key = ToIoHash(Id);
+		CbObjectWriter Record;
+		Record << "Key"sv << Id;
+
+		for (size_t Idx = 0; auto& Cid : Attachments)
+		{
+			Record.AddBinaryAttachment(fmt::format("attachment-{}", Idx++), Cid.second.DecodeRawHash());
+		}
+
+		IoBuffer Buffer = Record.Save().GetBuffer().AsIoBuffer();
+		Buffer.SetContentType(ZenContentType::kCbObject);
+		return {Key, Buffer};
+	}
+
 }  // namespace testutils
 
 TEST_CASE("z$.store")
@@ -1752,6 +1827,583 @@ TEST_CASE("z$.scrub")
 	CHECK(ScrubCtx.BadCids().GetSize() == 0);
 }
 
+TEST_CASE("z$.newgc.basics")
+{
+	using namespace testutils;
+
+	ScopedTemporaryDirectory TempDir;
+
+	auto JobQueue = MakeJobQueue(1, "testqueue");
+
+	struct CacheEntry
+	{
+		IoBuffer									  Data;
+		std::vector<std::pair<Oid, CompressedBuffer>> Attachments;
+	};
+
+	std::unordered_map<IoHash, CacheEntry> CacheEntries;
+
+	auto CreateCacheRecord =
+		[&](ZenCacheNamespace& Zcs, CidStore& CidStore, std::string_view Bucket, std::span<std::pair<Oid, CompressedBuffer>> Attachments) {
+			std::vector<IoHash> AttachmentKeys;
+			for (const auto& Attachment : Attachments)
+			{
+				AttachmentKeys.push_back(Attachment.second.DecodeRawHash());
+			}
+			auto Record = CreateRecord(Attachments);
+			Zcs.Put(Bucket,
+					Record.first,
+					{.Value	  = Record.second,
+					 .RawSize = Record.second.GetSize(),
+					 .RawHash = IoHash::HashBuffer(Record.second.GetData(), Record.second.GetSize())},
+					AttachmentKeys);
+			for (const auto& Attachment : Attachments)
+			{
+				CidStore.AddChunk(Attachment.second.GetCompressed().Flatten().AsIoBuffer(), Attachment.second.DecodeRawHash());
+			}
+			CacheEntries.insert({Record.first, CacheEntry{.Data = Record.second, .Attachments = {Attachments.begin(), Attachments.end()}}});
+			return Record.first;
+		};
+	auto CreateCacheValue = [&](ZenCacheNamespace& Zcs, std::string_view Bucket, size_t Size) {
+		std::pair<Oid, IoBuffer> CacheValue = CreateBinaryBlob(Size);
+		IoHash					 Key		= ToIoHash(CacheValue.first);
+		Zcs.Put(Bucket,
+				Key,
+				{.Value	  = CacheValue.second,
+				 .RawSize = CacheValue.second.GetSize(),
+				 .RawHash = IoHash::HashBuffer(CacheValue.second.GetData(), CacheValue.second.GetSize())},
+				{});
+		CacheEntries.insert({Key, CacheEntry{CacheValue.second, {}}});
+		return Key;
+	};
+
+	auto ValidateCacheEntry = [&](ZenCacheNamespace& Zcs,
+								  CidStore&			 CidStore,
+								  std::string_view	 Bucket,
+								  const IoHash&		 Key,
+								  bool				 ExpectCacheEntry,
+								  bool				 ExpectAttachments) {
+		const CacheEntry& Entry = CacheEntries[Key];
+		ZenCacheValue	  Value;
+		bool			  CacheExists = Zcs.Get(Bucket, Key, Value);
+		if (ExpectCacheEntry)
+		{
+			if (!CacheExists)
+			{
+				return false;
+			}
+			if (Value.Value.GetSize() != Entry.Data.GetSize())
+			{
+				return false;
+			}
+			if (!Value.Value.GetView().EqualBytes(Entry.Data.GetView()))
+			{
+				return false;
+			}
+		}
+		else if (CacheExists)
+		{
+			return false;
+		}
+
+		if (ExpectAttachments)
+		{
+			for (const auto& Attachment : Entry.Attachments)
+			{
+				IoHash	 AttachmentHash = Attachment.second.DecodeRawHash();
+				IoBuffer StoredData		= CidStore.FindChunkByCid(AttachmentHash);
+				if (!StoredData)
+				{
+					return false;
+				}
+				if (!StoredData.GetView().EqualBytes(Attachment.second.GetCompressed().Flatten().GetView()))
+				{
+					return false;
+				}
+			}
+		}
+		else
+		{
+			for (const auto& Attachment : Entry.Attachments)
+			{
+				IoHash AttachmentHash = Attachment.second.DecodeRawHash();
+				if (CidStore.ContainsChunk(AttachmentHash))
+				{
+					return false;
+				}
+			}
+		}
+		return true;
+	};
+
+	std::vector<IoHash> CacheRecords;
+	std::vector<IoHash> UnstructuredCacheValues;
+
+	const auto TearDrinkerBucket = "teardrinker"sv;
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		// Create some basic data
+		{
+			// Structured record with attachments
+			auto Attachments1 = CreateCompressedAttachment(CidStore, std::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87});
+			CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments1));
+
+			// Structured record with reuse of attachments
+			auto Attachments2 = CreateCompressedAttachment(CidStore, std::vector<size_t>{971});
+			Attachments2.push_back(Attachments1[0]);
+			Attachments2.push_back(Attachments1[1]);
+			CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments2));
+		}
+
+		CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, {}));
+
+		{
+			// Unstructured cache values
+			UnstructuredCacheValues.push_back(CreateCacheValue(Zcs, TearDrinkerBucket, 84));
+			UnstructuredCacheValues.push_back(CreateCacheValue(Zcs, TearDrinkerBucket, 591));
+			UnstructuredCacheValues.push_back(CreateCacheValue(Zcs, TearDrinkerBucket, 1024 * 1024 * 3 + 7));
+			UnstructuredCacheValues.push_back(CreateCacheValue(Zcs, TearDrinkerBucket, 71));
+		}
+	}
+
+	SUBCASE("expire nothing")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
+													   .CollectSmallObjects	   = false,
+													   .IsDeleteMode		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(0u, Result.ExpiredItems);
+		CHECK_EQ(0u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(0u, Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all large objects, delete nothing")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = false,
+													   .IsDeleteMode		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(1u, Result.ExpiredItems);
+		CHECK_EQ(0u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(0u, Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all object, delete nothing")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(7u, Result.ExpiredItems);
+		CHECK_EQ(0u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(0u, Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all large objects, skip cid")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = false,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = true});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(1u, Result.ExpiredItems);
+		CHECK_EQ(1u, Result.DeletedItems);
+		CHECK_EQ(0u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all objects, skip cid")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = true});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(7u, Result.ExpiredItems);
+		CHECK_EQ(7u, Result.DeletedItems);
+		CHECK_EQ(0u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_GE(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], false, true));
+	}
+	SUBCASE("expire all large objects")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = false,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(1u, Result.ExpiredItems);	// Only one cache value is pruned/deleted as that is the only large item in the cache (all other
+											// large items as in cas)
+		CHECK_EQ(1u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u,
+				 Result.PrunedReferences);	// We won't remove any references since all referencers are small which retains all references
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all objects")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		GcResult		  Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		= GcClock::Now() + std::chrono::minutes(1),
+																.ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+																.CollectSmallObjects	= true,
+																.IsDeleteMode			= true,
+																.SkipCidDelete			= false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(7u, Result.ExpiredItems);
+		CHECK_EQ(7u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(5u, Result.PrunedReferences);
+		CHECK_EQ(5u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, false));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], false, false));
+	}
+
+	SUBCASE("keep 1 cache record, skip cid")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = true});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(6u, Result.ExpiredItems);
+		CHECK_EQ(6u, Result.DeletedItems);
+		CHECK_EQ(0u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], false, false));
+	}
+
+	SUBCASE("keep 2 cache records")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(5u, Result.ExpiredItems);
+		CHECK_EQ(5u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, false));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], false, false));
+	}
+
+	SUBCASE("keep 3 cache value")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(4u, Result.ExpiredItems);
+		CHECK_EQ(4u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(5u, Result.PrunedReferences);
+		CHECK_EQ(5u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, false));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+
+	SUBCASE("keep 3 cache value, skip cid")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		// Prime so we can check GC of memory layer
+		ZenCacheValue Dummy;
+		Zcs.Get(TearDrinkerBucket, CacheRecords[0], Dummy);
+		Zcs.Get(TearDrinkerBucket, CacheRecords[1], Dummy);
+		Zcs.Get(TearDrinkerBucket, CacheRecords[2], Dummy);
+		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[0], Dummy);
+		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[1], Dummy);
+		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[2], Dummy);
+		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[3], Dummy);
+
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = true});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(4u, Result.ExpiredItems);
+		CHECK_EQ(4u, Result.DeletedItems);
+		CHECK_EQ(0u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		uint64_t MemoryClean = CacheEntries[CacheRecords[0]].Data.GetSize() + CacheEntries[CacheRecords[1]].Data.GetSize() +
+							   CacheEntries[CacheRecords[2]].Data.GetSize() + CacheEntries[UnstructuredCacheValues[0]].Data.GetSize();
+		CHECK_EQ(MemoryClean, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+
+	SUBCASE("leave write block")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		auto Attachments =
+			CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
+		IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments);
+
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::minutes(2));
+
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::minutes(2));
+
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = false});
+		CHECK_EQ(8u, Result.Items);
+		CHECK_EQ(1u, Result.ExpiredItems);
+		CHECK_EQ(1u, Result.DeletedItems);
+		CHECK_EQ(9u, Result.References);
+		CHECK_EQ(4u, Result.PrunedReferences);
+		CHECK_EQ(4u, Result.CompactedReferences);
+		CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(),
+				 Result.RemovedDiskSpace);
+		uint64_t MemoryClean = CacheEntries[CacheRecord].Data.GetSize();
+		CHECK_EQ(MemoryClean, Result.RemovedMemory);
+	}
+}
+
 #endif
 
 void