New GC implementation (#459)

- Feature: New garbage collection implementation, still in evaluation mode. Enabled by `--gc-v2` command line option

4 files changed, 1244 insertions, 104 deletions

diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 2efec1e66..38cbf3a93 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -168,8 +168,13 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
 const size_t ZenCacheDiskLayer::CacheBucket::UnknownReferencesIndex;
 const size_t ZenCacheDiskLayer::CacheBucket::NoReferencesIndex;
 
-ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName, const BucketConfiguration& Config)
-: m_BucketName(std::move(BucketName))
+ZenCacheDiskLayer::CacheBucket::CacheBucket(GcManager&				   Gc,
+											std::atomic_uint64_t&	   OuterCacheMemoryUsage,
+											std::string				   BucketName,
+											const BucketConfiguration& Config)
+: m_Gc(Gc)
+, m_OuterCacheMemoryUsage(OuterCacheMemoryUsage)
+, m_BucketName(std::move(BucketName))
 , m_Configuration(Config)
 , m_BucketId(Oid::Zero)
 {
@@ -179,10 +184,12 @@ ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName, const Bucket
 		// it makes sense to have a different strategy for legacy values
 		m_Configuration.LargeObjectThreshold = 16 * 1024 * 1024;
 	}
+	m_Gc.AddGcReferencer(*this);
 }
 
 ZenCacheDiskLayer::CacheBucket::~CacheBucket()
 {
+	m_Gc.RemoveGcReferencer(*this);
 }
 
 bool
@@ -717,7 +724,7 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(ZenContentType ContentTy
 }
 
 bool
-ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue, std::atomic_uint64_t& CacheMemoryUsage)
+ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue)
 {
 	metrics::RequestStats::Scope StatsScope(m_GetOps, 0);
 
@@ -782,8 +789,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 						if (!m_CachedPayloads[UpdateIt->second])
 						{
 							m_CachedPayloads[UpdateIt->second] = OutValue.Value;
-							m_MemCachedSize.fetch_add(ValueSize);
-							CacheMemoryUsage.fetch_add(ValueSize);
+							AddMemCacheUsage(ValueSize);
 							m_MemoryWriteCount++;
 						}
 					}
@@ -834,27 +840,24 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::Put(const IoHash&		  HashKey,
-									const ZenCacheValue&  Value,
-									std::span<IoHash>	  References,
-									std::atomic_uint64_t& CacheMemoryUsage)
+ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
 {
 	metrics::RequestStats::Scope $(m_PutOps, Value.Value.Size());
 
 	if (Value.Value.Size() >= m_Configuration.LargeObjectThreshold)
 	{
-		PutStandaloneCacheValue(HashKey, Value, References, CacheMemoryUsage);
+		PutStandaloneCacheValue(HashKey, Value, References);
 	}
 	else
 	{
-		PutInlineCacheValue(HashKey, Value, References, CacheMemoryUsage);
+		PutInlineCacheValue(HashKey, Value, References);
 	}
 
 	m_DiskWriteCount++;
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime, std::atomic_uint64_t& CacheMemoryUsage)
+ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime)
 {
 	GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count();
 
@@ -864,8 +867,7 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime, std:
 		if (m_AccessTimes[Kv.second] < ExpireTicks)
 		{
 			size_t PayloadSize = m_CachedPayloads[Kv.second].GetSize();
-			m_MemCachedSize.fetch_sub(PayloadSize);
-			CacheMemoryUsage.fetch_sub(PayloadSize);
+			RemoveMemCacheUsage(PayloadSize);
 			m_CachedPayloads[Kv.second] = {};
 		}
 	}
@@ -900,7 +902,7 @@ ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint		TickStart,
 }
 
 bool
-ZenCacheDiskLayer::CacheBucket::Drop(std::atomic_uint64_t& CacheMemoryUsage)
+ZenCacheDiskLayer::CacheBucket::Drop()
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::Drop");
 
@@ -926,7 +928,7 @@ ZenCacheDiskLayer::CacheBucket::Drop(std::atomic_uint64_t& CacheMemoryUsage)
 	m_NextReferenceHashesIndexes.clear();
 	m_ReferenceCount = 0;
 	m_StandaloneSize.store(0);
-	CacheMemoryUsage.fetch_sub(m_MemCachedSize.load());
+	m_OuterCacheMemoryUsage.fetch_sub(m_MemCachedSize.load());
 	m_MemCachedSize.store(0);
 
 	return Deleted;
@@ -1102,7 +1104,7 @@ ValidateCacheBucketEntryValue(ZenContentType ContentType, IoBuffer Buffer)
 };
 
 void
-ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx, std::atomic_uint64_t& CacheMemoryUsage)
+ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::Scrub");
 
@@ -1292,8 +1294,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx, std::atomic_uint
 					if (m_Configuration.MemCacheSizeThreshold > 0)
 					{
 						size_t CachedSize = m_CachedPayloads[It->second].GetSize();
-						m_MemCachedSize.fetch_sub(CachedSize);
-						CacheMemoryUsage.fetch_sub(CachedSize);
+						RemoveMemCacheUsage(CachedSize);
 						m_CachedPayloads[It->second] = IoBuffer{};
 					}
 
@@ -1411,8 +1412,9 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 
 	for (const auto& Entry : Index)
 	{
-		const IoHash& Key		 = Entry.first;
-		GcClock::Tick AccessTime = AccessTimes[Entry.second];
+		const IoHash& Key		   = Entry.first;
+		size_t		  PayloadIndex = Entry.second;
+		GcClock::Tick AccessTime   = AccessTimes[PayloadIndex];
 		if (AccessTime < ExpireTicks)
 		{
 			ExpiredKeys.push_back(Key);
@@ -1424,7 +1426,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 			continue;
 		}
 
-		BucketPayload&		Payload = Payloads[Entry.second];
+		BucketPayload&		Payload = Payloads[PayloadIndex];
 		const DiskLocation& Loc		= Payload.Location;
 
 		if (!Loc.IsFlagSet(DiskLocation::kStructured))
@@ -1433,7 +1435,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 		}
 		if (m_Configuration.EnableReferenceCaching)
 		{
-			if (FirstReferenceIndex.empty() || (FirstReferenceIndex[Entry.second] == UnknownReferencesIndex))
+			if (FirstReferenceIndex.empty() || (FirstReferenceIndex[PayloadIndex] == UnknownReferencesIndex))
 			{
 				StructuredItemsWithUnknownAttachments.push_back(Entry);
 				continue;
@@ -1450,7 +1452,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
 				});
 #endif	// CALCULATE_BLOCKING_TIME
-				if (auto It = m_Index.find(Entry.first); It != m_Index.end())
+				if (auto It = m_Index.find(Key); It != m_Index.end())
 				{
 					ReferencesAreKnown = GetReferences(IndexLock, m_FirstReferenceIndex[It->second], Cids);
 				}
@@ -1470,13 +1472,15 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 
 	for (const auto& Entry : StructuredItemsWithUnknownAttachments)
 	{
-		BucketPayload&		Payload = Payloads[Entry.second];
-		const DiskLocation& Loc		= Payload.Location;
+		const IoHash&		Key			 = Entry.first;
+		size_t				PayloadIndex = Entry.second;
+		BucketPayload&		Payload		 = Payloads[PayloadIndex];
+		const DiskLocation& Loc			 = Payload.Location;
 		{
 			IoBuffer Buffer;
 			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 			{
-				if (Buffer = GetStandaloneCacheValue(Loc.GetContentType(), Entry.first); !Buffer)
+				if (Buffer = GetStandaloneCacheValue(Loc.GetContentType(), Key); !Buffer)
 				{
 					continue;
 				}
@@ -1492,7 +1496,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
 				});
 #endif	// CALCULATE_BLOCKING_TIME
-				if (auto It = m_Index.find(Entry.first); It != m_Index.end())
+				if (auto It = m_Index.find(Key); It != m_Index.end())
 				{
 					if (m_Configuration.MemCacheSizeThreshold > 0)
 					{
@@ -1514,8 +1518,8 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 
 			ZEN_ASSERT(Buffer);
 			ZEN_ASSERT(Buffer.GetContentType() == ZenContentType::kCbObject);
-			CbObject Obj(SharedBuffer{Buffer});
-			size_t	 CurrentCidCount = Cids.size();
+			CbObjectView Obj(Buffer.GetData());
+			size_t		 CurrentCidCount = Cids.size();
 			Obj.IterateAttachments([&Cids](CbFieldView Field) { Cids.push_back(Field.AsAttachment()); });
 			if (m_Configuration.EnableReferenceCaching)
 			{
@@ -1528,7 +1532,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 					ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
 				});
 #endif	// CALCULATE_BLOCKING_TIME
-				if (auto It = m_Index.find(Entry.first); It != m_Index.end())
+				if (auto It = m_Index.find(Key); It != m_Index.end())
 				{
 					if (m_FirstReferenceIndex[It->second] == UnknownReferencesIndex)
 					{
@@ -1556,7 +1560,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx, std::atomic_uint64_t& CacheMemoryUsage)
+ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::CollectGarbage");
 
@@ -1762,17 +1766,19 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx, std::atomic_uin
 		TotalChunkCount = 0;
 		for (const auto& Entry : Index)
 		{
-			const DiskLocation& DiskLocation = Payloads[Entry.second].Location;
+			size_t				EntryIndex	 = Entry.second;
+			const DiskLocation& DiskLocation = Payloads[EntryIndex].Location;
 
 			if (DiskLocation.Flags & DiskLocation::kStandaloneFile)
 			{
 				continue;
 			}
+			const IoHash&	   Key		  = Entry.first;
 			BlockStoreLocation Location	  = DiskLocation.GetBlockLocation(m_Configuration.PayloadAlignment);
 			size_t			   ChunkIndex = ChunkLocations.size();
 			ChunkLocations.push_back(Location);
-			ChunkIndexToChunkHash[ChunkIndex] = Entry.first;
-			if (ExpiredCacheKeys.contains(Entry.first))
+			ChunkIndexToChunkHash[ChunkIndex] = Key;
+			if (ExpiredCacheKeys.contains(Key))
 			{
 				continue;
 			}
@@ -1815,12 +1821,12 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx, std::atomic_uin
 				});
 				for (const auto& Entry : MovedChunks)
 				{
-					size_t					  ChunkIndex   = Entry.first;
-					const BlockStoreLocation& NewLocation  = Entry.second;
-					const IoHash&			  ChunkHash	   = ChunkIndexToChunkHash[ChunkIndex];
-					size_t					  PayloadIndex = m_Index[ChunkHash];
-					BucketPayload&			  Payload	   = m_Payloads[PayloadIndex];
-					if (Payloads[Index[ChunkHash]].Location != m_Payloads[PayloadIndex].Location)
+					size_t					  ChunkIndex  = Entry.first;
+					const BlockStoreLocation& NewLocation = Entry.second;
+					const IoHash&			  ChunkHash	  = ChunkIndexToChunkHash[ChunkIndex];
+					size_t					  EntryIndex  = m_Index[ChunkHash];
+					BucketPayload&			  Payload	  = m_Payloads[EntryIndex];
+					if (Payloads[Index[ChunkHash]].Location != m_Payloads[EntryIndex].Location)
 					{
 						// Entry has been updated while GC was running, ignore the move
 						continue;
@@ -1830,9 +1836,9 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx, std::atomic_uin
 				}
 				for (const size_t ChunkIndex : RemovedChunks)
 				{
-					const IoHash&		 ChunkHash	  = ChunkIndexToChunkHash[ChunkIndex];
-					size_t				 PayloadIndex = m_Index[ChunkHash];
-					const BucketPayload& Payload	  = m_Payloads[PayloadIndex];
+					const IoHash&		 ChunkHash	= ChunkIndexToChunkHash[ChunkIndex];
+					size_t				 EntryIndex = m_Index[ChunkHash];
+					const BucketPayload& Payload	= m_Payloads[EntryIndex];
 					if (Payloads[Index[ChunkHash]].Location != Payload.Location)
 					{
 						// Entry has been updated while GC was running, ignore the delete
@@ -1843,12 +1849,11 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx, std::atomic_uin
 										  .Location = DiskLocation(OldDiskLocation.GetBlockLocation(m_Configuration.PayloadAlignment),
 																   m_Configuration.PayloadAlignment,
 																   OldDiskLocation.GetFlags() | DiskLocation::kTombStone)});
-					if (m_Configuration.MemCacheSizeThreshold > 0 && m_CachedPayloads[PayloadIndex])
+					if (m_Configuration.MemCacheSizeThreshold > 0 && m_CachedPayloads[EntryIndex])
 					{
-						uint64_t CachePayloadSize = m_CachedPayloads[PayloadIndex].Size();
-						m_MemCachedSize.fetch_sub(CachePayloadSize, std::memory_order::relaxed);
-						CacheMemoryUsage.fetch_sub(CachePayloadSize, std::memory_order::relaxed);
-						m_CachedPayloads[PayloadIndex] = IoBuffer{};
+						uint64_t CachePayloadSize = m_CachedPayloads[EntryIndex].Size();
+						RemoveMemCacheUsage(CachePayloadSize);
+						m_CachedPayloads[EntryIndex] = IoBuffer{};
 					}
 					m_Index.erase(ChunkHash);
 					DeletedChunks.insert(ChunkHash);
@@ -1891,10 +1896,10 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, size_t Index)
 	const BucketPayload& Payload = m_Payloads[Index];
 	if (Payload.Location.IsFlagSet(DiskLocation::kStructured))
 	{
-		IoBuffer Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile)
-							 ? GetStandaloneCacheValue(Payload.Location.GetContentType(), Key)
-							 : GetInlineCacheValue(Payload.Location);
-		CbObject Obj(SharedBuffer{Value});
+		IoBuffer	 Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile)
+								 ? GetStandaloneCacheValue(Payload.Location.GetContentType(), Key)
+								 : GetInlineCacheValue(Payload.Location);
+		CbObjectView Obj(Value.GetData());
 		Obj.IterateAttachments([&Attachments](CbFieldView Field) { Attachments.emplace_back(Field.AsAttachment()); });
 	}
 	return CacheValueDetails::ValueDetails{.Size		= Payload.Location.Size(),
@@ -1958,16 +1963,13 @@ ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx)
 	}
 	for (CacheBucket* Bucket : Buckets)
 	{
-		Bucket->CollectGarbage(GcCtx, m_TotalMemCachedSize);
+		Bucket->CollectGarbage(GcCtx);
 	}
 	MemCacheTrim(Buckets, GcCtx.CacheExpireTime());
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash&		  HashKey,
-														const ZenCacheValue&  Value,
-														std::span<IoHash>	  References,
-														std::atomic_uint64_t& CacheMemoryUsage)
+ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::PutStandaloneCacheValue");
 
@@ -2118,8 +2120,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash&		  HashKey
 			if (m_CachedPayloads[EntryIndex])
 			{
 				uint64_t CachePayloadSize = m_CachedPayloads[EntryIndex].Size();
-				m_MemCachedSize.fetch_sub(CachePayloadSize, std::memory_order::relaxed);
-				CacheMemoryUsage.fetch_sub(CachePayloadSize, std::memory_order::relaxed);
+				RemoveMemCacheUsage(CachePayloadSize);
 				m_CachedPayloads[EntryIndex] = IoBuffer{};
 			}
 		}
@@ -2131,10 +2132,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash&		  HashKey
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash&		  HashKey,
-													const ZenCacheValue&  Value,
-													std::span<IoHash>	  References,
-													std::atomic_uint64_t& CacheMemoryUsage)
+ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::PutInlineCacheValue");
 
@@ -2176,14 +2174,12 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash&		  HashKey,
 					if (m_CachedPayloads[EntryIndex])
 					{
 						uint64_t OldCachedSize = m_CachedPayloads[EntryIndex].GetSize();
-						m_MemCachedSize.fetch_sub(OldCachedSize);
-						CacheMemoryUsage.fetch_sub(OldCachedSize);
+						RemoveMemCacheUsage(OldCachedSize);
 					}
 
 					if (MemCacheBuffer)
 					{
-						m_MemCachedSize.fetch_add(PayloadSize);
-						CacheMemoryUsage.fetch_add(PayloadSize);
+						AddMemCacheUsage(PayloadSize);
 						m_MemoryWriteCount++;
 					}
 					m_CachedPayloads[EntryIndex] = std::move(MemCacheBuffer);
@@ -2202,8 +2198,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash&		  HashKey,
 				{
 					if (MemCacheBuffer)
 					{
-						m_MemCachedSize.fetch_add(PayloadSize);
-						CacheMemoryUsage.fetch_add(PayloadSize);
+						AddMemCacheUsage(PayloadSize);
 						m_MemoryWriteCount++;
 					}
 					m_CachedPayloads.emplace_back(std::move(MemCacheBuffer));
@@ -2219,6 +2214,409 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash&		  HashKey,
 }
 
 void
+ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx)
+{
+	size_t									 TotalEntries = 0;
+	tsl::robin_set<IoHash, IoHash::Hasher>	 ExpiredInlineKeys;
+	std::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys;
+
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] {
+		ZEN_DEBUG("gc cache bucket '{}': removed {} expired keys out of {} in {}",
+				  m_BucketDir,
+				  ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size(),
+				  TotalEntries,
+				  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	});
+
+	const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
+
+	BlockStoreCompactState			 BlockCompactState;
+	BlockStore::ReclaimSnapshotState BlockSnapshotState;
+	std::vector<IoHash>				 BlockCompactStateKeys;
+	std::vector<DiskIndexEntry>		 ExpiredEntries;
+	uint64_t						 RemovedStandaloneSize = 0;
+	{
+		RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
+		if (Ctx.Settings.CollectSmallObjects)
+		{
+			BlockSnapshotState = m_BlockStore.GetReclaimSnapshotState();
+		}
+		TotalEntries = m_Index.size();
+
+		// Find out expired keys and affected blocks
+		for (const auto& Entry : m_Index)
+		{
+			const IoHash& Key		 = Entry.first;
+			size_t		  EntryIndex = Entry.second;
+			GcClock::Tick AccessTime = m_AccessTimes[EntryIndex];
+			if (AccessTime >= ExpireTicks)
+			{
+				continue;
+			}
+
+			const BucketPayload& Payload	  = m_Payloads[EntryIndex];
+			DiskIndexEntry		 ExpiredEntry = {.Key = Key, .Location = Payload.Location};
+			ExpiredEntry.Location.Flags |= DiskLocation::kTombStone;
+
+			if (Payload.Location.Flags & DiskLocation::kStandaloneFile)
+			{
+				ExpiredStandaloneKeys.push_back({Key, Payload.Location.Size()});
+				RemovedStandaloneSize += Payload.Location.Size();
+				ExpiredEntries.push_back(ExpiredEntry);
+			}
+			else if (Ctx.Settings.CollectSmallObjects)
+			{
+				ExpiredInlineKeys.insert(Key);
+				uint32_t BlockIndex			= Payload.Location.Location.BlockLocation.GetBlockIndex();
+				bool	 IsActiveWriteBlock = BlockSnapshotState.m_ActiveWriteBlocks.contains(BlockIndex);
+				if (!IsActiveWriteBlock)
+				{
+					BlockCompactState.AddBlock(BlockIndex);
+				}
+				ExpiredEntries.push_back(ExpiredEntry);
+			}
+		}
+
+		Ctx.ExpiredItems.fetch_add(ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size());
+
+		// Get all locations we need to keep for affected blocks
+		if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
+		{
+			for (const auto& Entry : m_Index)
+			{
+				const IoHash& Key = Entry.first;
+				if (ExpiredInlineKeys.contains(Key))
+				{
+					continue;
+				}
+				size_t				 EntryIndex = Entry.second;
+				const BucketPayload& Payload	= m_Payloads[EntryIndex];
+				if (Payload.Location.Flags & DiskLocation::kStandaloneFile)
+				{
+					continue;
+				}
+				if (BlockCompactState.AddKeepLocation(Payload.Location.GetBlockLocation(m_Configuration.PayloadAlignment)))
+				{
+					BlockCompactStateKeys.push_back(Key);
+				}
+			}
+		}
+
+		if (Ctx.Settings.IsDeleteMode)
+		{
+			for (const DiskIndexEntry& Entry : ExpiredEntries)
+			{
+				auto It = m_Index.find(Entry.Key);
+				ZEN_ASSERT(It != m_Index.end());
+				if (m_Configuration.MemCacheSizeThreshold > 0 && m_CachedPayloads[It->second])
+				{
+					size_t PayloadSize = m_CachedPayloads[It->second].GetSize();
+					Ctx.RemovedMemory.fetch_add(PayloadSize);
+					RemoveMemCacheUsage(PayloadSize);
+				}
+				m_Index.erase(It);
+			}
+			m_SlogFile.Append(ExpiredEntries);
+			m_StandaloneSize.fetch_sub(RemovedStandaloneSize, std::memory_order::relaxed);
+		}
+	}
+	Ctx.Items.fetch_add(TotalEntries);
+
+	if (ExpiredEntries.empty())
+	{
+		return;
+	}
+
+	if (!Ctx.Settings.IsDeleteMode)
+	{
+		return;
+	}
+
+	Ctx.DeletedItems.fetch_add(ExpiredEntries.size());
+
+	// Compact standalone items
+	ExtendablePathBuilder<256> Path;
+	for (const std::pair<IoHash, uint64_t>& ExpiredKey : ExpiredStandaloneKeys)
+	{
+		Path.Reset();
+		BuildPath(Path, ExpiredKey.first);
+		fs::path FilePath = Path.ToPath();
+
+		RwLock::SharedLockScope IndexLock(m_IndexLock);
+		if (m_Index.contains(ExpiredKey.first))
+		{
+			// Someone added it back, let the file on disk be
+			ZEN_DEBUG("gc cache bucket '{}': skipping z$ delete standalone of file '{}' FAILED, it has been added back",
+					  m_BucketDir,
+					  Path.ToUtf8());
+			continue;
+		}
+
+		RwLock::ExclusiveLockScope ValueLock(LockForHash(ExpiredKey.first));
+		IndexLock.ReleaseNow();
+		ZEN_DEBUG("gc cache bucket '{}': deleting standalone cache file '{}'", m_BucketDir, Path.ToUtf8());
+
+		std::error_code Ec;
+		if (!fs::remove(FilePath, Ec))
+		{
+			continue;
+		}
+		if (Ec)
+		{
+			ZEN_WARN("gc cache bucket '{}': delete expired z$ standalone file '{}' FAILED, reason: '{}'",
+					 m_BucketDir,
+					 Path.ToUtf8(),
+					 Ec.message());
+			continue;
+		}
+		Ctx.RemovedDiskSpace.fetch_add(ExpiredKey.second);
+	}
+
+	if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
+	{
+		// Compact block store
+		m_BlockStore.CompactBlocks(
+			BlockCompactState,
+			m_Configuration.PayloadAlignment,
+			[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
+				std::vector<DiskIndexEntry> MovedEntries;
+				RwLock::ExclusiveLockScope	_(m_IndexLock);
+				for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
+				{
+					size_t		  ChunkIndex = Moved.first;
+					const IoHash& Key		 = BlockCompactStateKeys[ChunkIndex];
+
+					if (auto It = m_Index.find(Key); It != m_Index.end())
+					{
+						BucketPayload&			  Payload	  = m_Payloads[It->second];
+						const BlockStoreLocation& OldLocation = BlockCompactState.GetLocation(ChunkIndex);
+						if (Payload.Location.GetBlockLocation(m_Configuration.PayloadAlignment) != OldLocation)
+						{
+							// Someone has moved our chunk so lets just skip the new location we were provided, it will be GC:d at a later
+							// time
+							continue;
+						}
+
+						const BlockStoreLocation& NewLocation = Moved.second;
+
+						Payload.Location = DiskLocation(NewLocation, m_Configuration.PayloadAlignment, Payload.Location.GetFlags());
+						MovedEntries.push_back({.Key = Key, .Location = Payload.Location});
+					}
+				}
+				m_SlogFile.Append(MovedEntries);
+				Ctx.RemovedDiskSpace.fetch_add(FreedDiskSpace);
+			},
+			[&]() { return 0; });
+	}
+
+	std::vector<BucketPayload> Payloads;
+	std::vector<AccessTime>	   AccessTimes;
+	std::vector<IoBuffer>	   CachedPayloads;
+	std::vector<size_t>		   FirstReferenceIndex;
+	IndexMap				   Index;
+	{
+		RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
+		CompactState(Payloads, AccessTimes, CachedPayloads, FirstReferenceIndex, Index, IndexLock);
+	}
+}
+
+class DiskBucketReferenceChecker : public GcReferenceChecker
+{
+public:
+	DiskBucketReferenceChecker(ZenCacheDiskLayer::CacheBucket& Owner) : m_CacheBucket(Owner) {}
+
+	virtual ~DiskBucketReferenceChecker()
+	{
+		m_IndexLock.reset();
+		if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
+		{
+			// If reference caching is not enabled, we temporarily used the data structure for reference caching, lets reset it
+			m_CacheBucket.ClearReferenceCache();
+		}
+	}
+
+	virtual void LockState(GcCtx&) override
+	{
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			ZEN_DEBUG("gc cache bucket '{}': found {} references in {}",
+					  m_CacheBucket.m_BucketDir,
+					  m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
+					  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		m_IndexLock = std::make_unique<RwLock::SharedLockScope>(m_CacheBucket.m_IndexLock);
+
+		// Rescan to see if any cache items needs refreshing since last pass when we had the lock
+		for (const auto& Entry : m_CacheBucket.m_Index)
+		{
+			size_t												 PayloadIndex = Entry.second;
+			const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload	  = m_CacheBucket.m_Payloads[PayloadIndex];
+			const DiskLocation&									 Loc		  = Payload.Location;
+
+			if (!Loc.IsFlagSet(DiskLocation::kStructured))
+			{
+				continue;
+			}
+			ZEN_ASSERT(!m_CacheBucket.m_FirstReferenceIndex.empty());
+			const IoHash& Key = Entry.first;
+			if (m_CacheBucket.m_FirstReferenceIndex[PayloadIndex] == ZenCacheDiskLayer::CacheBucket::UnknownReferencesIndex)
+			{
+				IoBuffer Buffer;
+				if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
+				{
+					Buffer = m_CacheBucket.GetStandaloneCacheValue(Loc.GetContentType(), Key);
+				}
+				else
+				{
+					Buffer = m_CacheBucket.GetInlineCacheValue(Loc);
+				}
+
+				if (Buffer)
+				{
+					ZEN_ASSERT(Buffer.GetContentType() == ZenContentType::kCbObject);
+					CbObjectView Obj(Buffer.GetData());
+					Obj.IterateAttachments([this](CbFieldView Field) { m_UncachedReferences.insert(Field.AsAttachment()); });
+				}
+			}
+		}
+	}
+
+	virtual void RemoveUsedReferencesFromSet(GcCtx&, HashSet& IoCids) override
+	{
+		ZEN_ASSERT(m_IndexLock);
+
+		for (const IoHash& ReferenceHash : m_CacheBucket.m_ReferenceHashes)
+		{
+			IoCids.erase(ReferenceHash);
+		}
+
+		for (const IoHash& ReferenceHash : m_UncachedReferences)
+		{
+			IoCids.erase(ReferenceHash);
+		}
+	}
+	ZenCacheDiskLayer::CacheBucket&			 m_CacheBucket;
+	std::unique_ptr<RwLock::SharedLockScope> m_IndexLock;
+	HashSet									 m_UncachedReferences;
+};
+
+std::vector<GcReferenceChecker*>
+ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx&)
+{
+	Stopwatch  Timer;
+	const auto _ = MakeGuard(
+		[&] { ZEN_DEBUG("gc cache bucket '{}': refreshed reference cache in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+
+	std::vector<IoHash> UpdateKeys;
+	std::vector<IoHash> StandaloneKeys;
+	std::vector<size_t> ReferenceCounts;
+	std::vector<IoHash> References;
+
+	// Refresh cache
+	{
+		RwLock::SharedLockScope IndexLock(m_IndexLock);
+		for (const auto& Entry : m_Index)
+		{
+			size_t												 PayloadIndex = Entry.second;
+			const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload	  = m_Payloads[PayloadIndex];
+			const DiskLocation&									 Loc		  = Payload.Location;
+
+			if (!Loc.IsFlagSet(DiskLocation::kStructured))
+			{
+				continue;
+			}
+			if (m_Configuration.EnableReferenceCaching &&
+				m_FirstReferenceIndex[PayloadIndex] != ZenCacheDiskLayer::CacheBucket::UnknownReferencesIndex)
+			{
+				continue;
+			}
+			const IoHash& Key = Entry.first;
+			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
+			{
+				StandaloneKeys.push_back(Key);
+				continue;
+			}
+			IoBuffer Buffer = GetInlineCacheValue(Loc);
+			if (!Buffer)
+			{
+				UpdateKeys.push_back(Key);
+				ReferenceCounts.push_back(0);
+				continue;
+			}
+			size_t CurrentReferenceCount = References.size();
+			{
+				CbObjectView Obj(Buffer.GetData());
+				Obj.IterateAttachments([&References](CbFieldView Field) { References.emplace_back(Field.AsAttachment()); });
+				Buffer = {};
+			}
+			UpdateKeys.push_back(Key);
+			ReferenceCounts.push_back(References.size() - CurrentReferenceCount);
+		}
+	}
+	{
+		for (const IoHash& Key : StandaloneKeys)
+		{
+			IoBuffer Buffer = GetStandaloneCacheValue(ZenContentType::kCbObject, Key);
+			if (!Buffer)
+			{
+				continue;
+			}
+
+			size_t CurrentReferenceCount = References.size();
+			{
+				CbObjectView Obj(Buffer.GetData());
+				Obj.IterateAttachments([&References](CbFieldView Field) { References.emplace_back(Field.AsAttachment()); });
+				Buffer = {};
+			}
+			UpdateKeys.push_back(Key);
+			ReferenceCounts.push_back(References.size() - CurrentReferenceCount);
+		}
+	}
+
+	{
+		size_t					   ReferenceOffset = 0;
+		RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
+		if (!m_Configuration.EnableReferenceCaching)
+		{
+			ZEN_ASSERT(m_FirstReferenceIndex.empty());
+			ZEN_ASSERT(m_ReferenceHashes.empty());
+			ZEN_ASSERT(m_NextReferenceHashesIndexes.empty());
+			ZEN_ASSERT(m_ReferenceCount == 0);
+			// If reference caching is not enabled, we will resize and use the data structure in place for reference caching when
+			// we figure out what this bucket references. This will be reset once the DiskBucketReferenceChecker is deleted.
+			m_FirstReferenceIndex.resize(m_Payloads.size(), UnknownReferencesIndex);
+		}
+		for (size_t Index = 0; Index < UpdateKeys.size(); Index++)
+		{
+			const IoHash& Key			 = UpdateKeys[Index];
+			size_t		  ReferenceCount = ReferenceCounts[Index];
+			auto		  It			 = m_Index.find(Key);
+			if (It == m_Index.end())
+			{
+				ReferenceOffset += ReferenceCount;
+				continue;
+			}
+			if (m_FirstReferenceIndex[It->second] != ZenCacheDiskLayer::CacheBucket::UnknownReferencesIndex)
+			{
+				continue;
+			}
+			SetReferences(IndexLock,
+						  m_FirstReferenceIndex[It->second],
+						  std::span<IoHash>{References.data() + ReferenceOffset, ReferenceCount});
+			ReferenceOffset += ReferenceCount;
+		}
+		if (m_Configuration.EnableReferenceCaching)
+		{
+			CompactReferences(IndexLock);
+		}
+	}
+
+	return {new DiskBucketReferenceChecker(*this)};
+}
+
+void
 ZenCacheDiskLayer::CacheBucket::CompactReferences(RwLock::ExclusiveLockScope&)
 {
 	std::vector<size_t> FirstReferenceIndex;
@@ -2381,6 +2779,19 @@ ZenCacheDiskLayer::CacheBucket::LockedGetReferences(std::size_t FirstReferenceIn
 }
 
 void
+ZenCacheDiskLayer::CacheBucket::ClearReferenceCache()
+{
+	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
+	m_FirstReferenceIndex.clear();
+	m_FirstReferenceIndex.shrink_to_fit();
+	m_ReferenceHashes.clear();
+	m_ReferenceHashes.shrink_to_fit();
+	m_NextReferenceHashesIndexes.clear();
+	m_NextReferenceHashesIndexes.shrink_to_fit();
+	m_ReferenceCount = 0;
+}
+
+void
 ZenCacheDiskLayer::CacheBucket::CompactState(std::vector<BucketPayload>& Payloads,
 											 std::vector<AccessTime>&	 AccessTimes,
 											 std::vector<IoBuffer>&		 CachedPayloads,
@@ -2426,16 +2837,34 @@ ZenCacheDiskLayer::CacheBucket::CompactState(std::vector<BucketPayload>& Payload
 	}
 }
 
+#if ZEN_WITH_TESTS
+void
+ZenCacheDiskLayer::CacheBucket::SetAccessTime(const IoHash& HashKey, GcClock::TimePoint Time)
+{
+	GcClock::Tick			TimeTick = Time.time_since_epoch().count();
+	RwLock::SharedLockScope IndexLock(m_IndexLock);
+	if (auto It = m_Index.find(HashKey); It != m_Index.end())
+	{
+		size_t EntryIndex = It.value();
+		ZEN_ASSERT_SLOW(EntryIndex < m_AccessTimes.size());
+		m_AccessTimes[EntryIndex] = TimeTick;
+	}
+}
+#endif	// ZEN_WITH_TESTS
+
 //////////////////////////////////////////////////////////////////////////
 
-ZenCacheDiskLayer::ZenCacheDiskLayer(JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config)
-: m_JobQueue(JobQueue)
+ZenCacheDiskLayer::ZenCacheDiskLayer(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config)
+: m_Gc(Gc)
+, m_JobQueue(JobQueue)
 , m_RootDir(RootDir)
 , m_Configuration(Config)
 {
 }
 
-ZenCacheDiskLayer::~ZenCacheDiskLayer() = default;
+ZenCacheDiskLayer::~ZenCacheDiskLayer()
+{
+}
 
 bool
 ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
@@ -2468,8 +2897,10 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 		}
 		else
 		{
-			auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName, m_Configuration.BucketConfig));
-			Bucket			  = InsertResult.first->second.get();
+			auto InsertResult =
+				m_Buckets.emplace(BucketName,
+								  std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig));
+			Bucket = InsertResult.first->second.get();
 
 			std::filesystem::path BucketPath = m_RootDir;
 			BucketPath /= BucketName;
@@ -2483,7 +2914,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 	}
 
 	ZEN_ASSERT(Bucket != nullptr);
-	if (Bucket->Get(HashKey, OutValue, m_TotalMemCachedSize))
+	if (Bucket->Get(HashKey, OutValue))
 	{
 		TryMemCacheTrim();
 		return true;
@@ -2522,8 +2953,10 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 		}
 		else
 		{
-			auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName, m_Configuration.BucketConfig));
-			Bucket			  = InsertResult.first->second.get();
+			auto InsertResult =
+				m_Buckets.emplace(BucketName,
+								  std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig));
+			Bucket = InsertResult.first->second.get();
 
 			std::filesystem::path BucketPath = m_RootDir;
 			BucketPath /= BucketName;
@@ -2547,7 +2980,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 
 	ZEN_ASSERT(Bucket != nullptr);
 
-	Bucket->Put(HashKey, Value, References, m_TotalMemCachedSize);
+	Bucket->Put(HashKey, Value, References);
 	TryMemCacheTrim();
 }
 
@@ -2579,8 +3012,10 @@ ZenCacheDiskLayer::DiscoverBuckets()
 			continue;
 		}
 
-		auto		 InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName, m_Configuration.BucketConfig));
-		CacheBucket& Bucket		  = *InsertResult.first->second;
+		auto InsertResult =
+			m_Buckets.emplace(BucketName,
+							  std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig));
+		CacheBucket& Bucket = *InsertResult.first->second;
 
 		try
 		{
@@ -2636,7 +3071,7 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket)
 		m_DroppedBuckets.push_back(std::move(It->second));
 		m_Buckets.erase(It);
 
-		return Bucket.Drop(m_TotalMemCachedSize);
+		return Bucket.Drop();
 	}
 
 	// Make sure we remove the folder even if we don't know about the bucket
@@ -2658,7 +3093,7 @@ ZenCacheDiskLayer::Drop()
 		CacheBucket& Bucket = *It->second;
 		m_DroppedBuckets.push_back(std::move(It->second));
 		m_Buckets.erase(It->first);
-		if (!Bucket.Drop(m_TotalMemCachedSize))
+		if (!Bucket.Drop())
 		{
 			return false;
 		}
@@ -2700,10 +3135,10 @@ ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx)
 		{
 #if 1
 			Results.push_back(Ctx.ThreadPool().EnqueueTask(
-				std::packaged_task<void()>{[this, Bucket = Kv.second.get(), &Ctx] { Bucket->ScrubStorage(Ctx, m_TotalMemCachedSize); }}));
+				std::packaged_task<void()>{[this, Bucket = Kv.second.get(), &Ctx] { Bucket->ScrubStorage(Ctx); }}));
 #else
 			CacheBucket& Bucket = *Kv.second;
-			Bucket.ScrubStorage(Ctx, m_TotalMemCachedSize);
+			Bucket.ScrubStorage(Ctx);
 #endif
 		}
 
@@ -2914,7 +3349,7 @@ ZenCacheDiskLayer::MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::Tim
 	RwLock::SharedLockScope __(m_Lock);
 	for (CacheBucket* Bucket : Buckets)
 	{
-		Bucket->MemCacheTrim(ExpireTime, m_TotalMemCachedSize);
+		Bucket->MemCacheTrim(ExpireTime);
 	}
 	const GcClock::TimePoint   Now				   = GcClock::Now();
 	const GcClock::Tick		   NowTick			   = Now.time_since_epoch().count();
@@ -2924,4 +3359,30 @@ ZenCacheDiskLayer::MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::Tim
 	m_LastTickMemCacheTrim.compare_exchange_strong(LastTrimTick, NextAllowedTrimTick);
 }
 
+#if ZEN_WITH_TESTS
+void
+ZenCacheDiskLayer::SetAccessTime(std::string_view InBucket, const IoHash& HashKey, GcClock::TimePoint Time)
+{
+	const auto	 BucketName = std::string(InBucket);
+	CacheBucket* Bucket		= nullptr;
+
+	{
+		RwLock::SharedLockScope _(m_Lock);
+
+		auto It = m_Buckets.find(BucketName);
+
+		if (It != m_Buckets.end())
+		{
+			Bucket = It->second.get();
+		}
+	}
+
+	if (Bucket == nullptr)
+	{
+		return;
+	}
+	Bucket->SetAccessTime(HashKey, Time);
+}
+#endif	// ZEN_WITH_TESTS
+
 }  // namespace zen
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index cc6653e28..d8f51c398 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -151,7 +151,7 @@ public:
 		uint64_t					  MemorySize;
 	};
 
-	explicit ZenCacheDiskLayer(JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config);
+	explicit ZenCacheDiskLayer(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config);
 	~ZenCacheDiskLayer();
 
 	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -174,24 +174,28 @@ public:
 
 	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
 
+#if ZEN_WITH_TESTS
+	void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time);
+#endif	// ZEN_WITH_TESTS
+
 private:
 	/** A cache bucket manages a single directory containing
 		metadata and data for that bucket
 	  */
-	struct CacheBucket
+	struct CacheBucket : public GcReferencer
 	{
-		CacheBucket(std::string BucketName, const BucketConfiguration& Config);
+		CacheBucket(GcManager& Gc, std::atomic_uint64_t& OuterCacheMemoryUsage, std::string BucketName, const BucketConfiguration& Config);
 		~CacheBucket();
 
 		bool OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
-		bool Get(const IoHash& HashKey, ZenCacheValue& OutValue, std::atomic_uint64_t& CacheMemoryUsage);
-		void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, std::atomic_uint64_t& CacheMemoryUsage);
-		void MemCacheTrim(GcClock::TimePoint ExpireTime, std::atomic_uint64_t& CacheMemoryUsage);
-		bool Drop(std::atomic_uint64_t& CacheMemoryUsage);
+		bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
+		void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
+		void MemCacheTrim(GcClock::TimePoint ExpireTime);
+		bool Drop();
 		void Flush();
-		void ScrubStorage(ScrubContext& Ctx, std::atomic_uint64_t& CacheMemoryUsage);
+		void ScrubStorage(ScrubContext& Ctx);
 		void GatherReferences(GcContext& GcCtx);
-		void CollectGarbage(GcContext& GcCtx, std::atomic_uint64_t& CacheMemoryUsage);
+		void CollectGarbage(GcContext& GcCtx);
 
 		inline GcStorageSize StorageSize() const
 		{
@@ -205,8 +209,13 @@ private:
 		void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
 
 		void GetUsageByAccess(GcClock::TimePoint TickStart, GcClock::Duration SectionLength, std::vector<uint64_t>& InOutUsageSlots);
+#if ZEN_WITH_TESTS
+		void SetAccessTime(const IoHash& HashKey, GcClock::TimePoint Time);
+#endif	// ZEN_WITH_TESTS
 
 	private:
+		GcManager&			  m_Gc;
+		std::atomic_uint64_t& m_OuterCacheMemoryUsage;
 		std::string			  m_BucketName;
 		std::filesystem::path m_BucketDir;
 		std::filesystem::path m_BlocksBasePath;
@@ -258,16 +267,13 @@ private:
 		std::atomic_uint64_t	   m_StandaloneSize{};
 		std::atomic_uint64_t	   m_MemCachedSize{};
 
+		virtual void							 RemoveExpiredData(GcCtx& Ctx) override;
+		virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
+
 		void	 BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
-		void	 PutStandaloneCacheValue(const IoHash&		   HashKey,
-										 const ZenCacheValue&  Value,
-										 std::span<IoHash>	   References,
-										 std::atomic_uint64_t& CacheMemoryUsage);
+		void	 PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 		IoBuffer GetStandaloneCacheValue(ZenContentType ContentType, const IoHash& HashKey) const;
-		void	 PutInlineCacheValue(const IoHash&		   HashKey,
-									 const ZenCacheValue&  Value,
-									 std::span<IoHash>	   References,
-									 std::atomic_uint64_t& CacheMemoryUsage);
+		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 		IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const;
 		void	 MakeIndexSnapshot();
 		uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
@@ -289,6 +295,7 @@ private:
 		}
 		size_t AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key);
 		bool   LockedGetReferences(std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const;
+		void   ClearReferenceCache();
 
 		void CompactState(std::vector<BucketPayload>& TmpPayloads,
 						  std::vector<AccessTime>&	  TmpAccessTimes,
@@ -297,6 +304,17 @@ private:
 						  IndexMap&					  TmpIndex,
 						  RwLock::ExclusiveLockScope& IndexLock);
 
+		void AddMemCacheUsage(uint64_t ValueSize)
+		{
+			m_MemCachedSize.fetch_add(ValueSize, std::memory_order::relaxed);
+			m_OuterCacheMemoryUsage.fetch_add(ValueSize, std::memory_order::relaxed);
+		}
+		void RemoveMemCacheUsage(uint64_t ValueSize)
+		{
+			m_MemCachedSize.fetch_sub(ValueSize, std::memory_order::relaxed);
+			m_OuterCacheMemoryUsage.fetch_sub(ValueSize, std::memory_order::relaxed);
+		}
+
 		// These locks are here to avoid contention on file creation, therefore it's sufficient
 		// that we take the same lock for the same hash
 		//
@@ -305,6 +323,8 @@ private:
 		// an issue in practice
 		mutable RwLock m_ShardedLocks[256];
 		inline RwLock& LockForHash(const IoHash& Hash) const { return m_ShardedLocks[Hash.Hash[19]]; }
+
+		friend class DiskBucketReferenceChecker;
 	};
 
 	inline void TryMemCacheTrim()
@@ -326,6 +346,7 @@ private:
 	void MemCacheTrim();
 	void MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime);
 
+	GcManager&													  m_Gc;
 	JobQueue&													  m_JobQueue;
 	std::filesystem::path										  m_RootDir;
 	Configuration												  m_Configuration;
@@ -338,6 +359,8 @@ private:
 
 	ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete;
 	ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;
+
+	friend class DiskBucketReferenceChecker;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 6fab14eee..516532528 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -64,7 +64,7 @@ ZenCacheNamespace::ZenCacheNamespace(GcManager& Gc, JobQueue& JobQueue, const st
 , m_JobQueue(JobQueue)
 , m_RootDir(RootDir)
 , m_Configuration(Config)
-, m_DiskLayer(m_JobQueue, m_RootDir, m_Configuration.DiskLayerConfig)
+, m_DiskLayer(m_Gc, m_JobQueue, m_RootDir, m_Configuration.DiskLayerConfig)
 {
 	ZEN_INFO("initializing structured cache at '{}'", m_RootDir);
 	CreateDirectories(m_RootDir);
@@ -232,6 +232,14 @@ ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const st
 	return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter);
 }
 
+#if ZEN_WITH_TESTS
+void
+ZenCacheNamespace::SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time)
+{
+	m_DiskLayer.SetAccessTime(Bucket, HashKey, Time);
+}
+#endif	// ZEN_WITH_TESTS
+
 //////////////////////////// ZenCacheStore
 
 ZEN_DEFINE_LOG_CATEGORY_STATIC(LogCacheActivity, "z$");
@@ -784,6 +792,73 @@ namespace testutils {
 		return Buf;
 	};
 
+	IoHash ToIoHash(const Oid& Id)
+	{
+		char OIdString[24 + 1];
+		Id.ToString(OIdString);
+		IoHash Key = IoHash::HashBuffer(OIdString, 24);
+		return Key;
+	}
+
+	std::pair<Oid, IoBuffer> CreateBinaryBlob(size_t Size)
+	{
+		uint64_t seed{Size};
+		auto	 next = [](uint64_t& seed) {
+			uint64_t z = (seed += UINT64_C(0x9E3779B97F4A7C15));
+			z		   = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4E5B9);
+			z		   = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB);
+			return z ^ (z >> 31);
+		};
+
+		IoBuffer  Data(Size);
+		uint64_t* DataPtr = reinterpret_cast<uint64_t*>(Data.MutableData());
+		while (Size > sizeof(uint64_t))
+		{
+			*DataPtr++ = next(seed);
+			Size -= sizeof(uint64_t);
+		}
+		uint64_t ByteNext	 = next(seed);
+		uint8_t* ByteDataPtr = reinterpret_cast<uint8_t*>(DataPtr);
+		while (Size > 0)
+		{
+			*ByteDataPtr++ = static_cast<uint8_t>(ByteNext & 0xff);
+			ByteNext >>= 8;
+			Size--;
+		}
+		return {Oid::NewOid(), Data};
+	}
+
+	std::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes)
+	{
+		std::vector<std::pair<Oid, CompressedBuffer>> Result;
+		Result.reserve(Sizes.size());
+		for (size_t Size : Sizes)
+		{
+			auto			 Blob		= CreateBinaryBlob(Size);
+			CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(Blob.second.Data(), Blob.second.Size()));
+			CHECK(!Store.ContainsChunk(Compressed.DecodeRawHash()));
+			Result.emplace_back(std::pair<Oid, CompressedBuffer>(Blob.first, Compressed));
+		}
+		return Result;
+	}
+
+	std::pair<IoHash, IoBuffer> CreateRecord(std::span<std::pair<Oid, CompressedBuffer>> Attachments)
+	{
+		Oid			   Id  = Oid::NewOid();
+		IoHash		   Key = ToIoHash(Id);
+		CbObjectWriter Record;
+		Record << "Key"sv << Id;
+
+		for (size_t Idx = 0; auto& Cid : Attachments)
+		{
+			Record.AddBinaryAttachment(fmt::format("attachment-{}", Idx++), Cid.second.DecodeRawHash());
+		}
+
+		IoBuffer Buffer = Record.Save().GetBuffer().AsIoBuffer();
+		Buffer.SetContentType(ZenContentType::kCbObject);
+		return {Key, Buffer};
+	}
+
 }  // namespace testutils
 
 TEST_CASE("z$.store")
@@ -1752,6 +1827,583 @@ TEST_CASE("z$.scrub")
 	CHECK(ScrubCtx.BadCids().GetSize() == 0);
 }
 
+TEST_CASE("z$.newgc.basics")
+{
+	using namespace testutils;
+
+	ScopedTemporaryDirectory TempDir;
+
+	auto JobQueue = MakeJobQueue(1, "testqueue");
+
+	struct CacheEntry
+	{
+		IoBuffer									  Data;
+		std::vector<std::pair<Oid, CompressedBuffer>> Attachments;
+	};
+
+	std::unordered_map<IoHash, CacheEntry> CacheEntries;
+
+	auto CreateCacheRecord =
+		[&](ZenCacheNamespace& Zcs, CidStore& CidStore, std::string_view Bucket, std::span<std::pair<Oid, CompressedBuffer>> Attachments) {
+			std::vector<IoHash> AttachmentKeys;
+			for (const auto& Attachment : Attachments)
+			{
+				AttachmentKeys.push_back(Attachment.second.DecodeRawHash());
+			}
+			auto Record = CreateRecord(Attachments);
+			Zcs.Put(Bucket,
+					Record.first,
+					{.Value	  = Record.second,
+					 .RawSize = Record.second.GetSize(),
+					 .RawHash = IoHash::HashBuffer(Record.second.GetData(), Record.second.GetSize())},
+					AttachmentKeys);
+			for (const auto& Attachment : Attachments)
+			{
+				CidStore.AddChunk(Attachment.second.GetCompressed().Flatten().AsIoBuffer(), Attachment.second.DecodeRawHash());
+			}
+			CacheEntries.insert({Record.first, CacheEntry{.Data = Record.second, .Attachments = {Attachments.begin(), Attachments.end()}}});
+			return Record.first;
+		};
+	auto CreateCacheValue = [&](ZenCacheNamespace& Zcs, std::string_view Bucket, size_t Size) {
+		std::pair<Oid, IoBuffer> CacheValue = CreateBinaryBlob(Size);
+		IoHash					 Key		= ToIoHash(CacheValue.first);
+		Zcs.Put(Bucket,
+				Key,
+				{.Value	  = CacheValue.second,
+				 .RawSize = CacheValue.second.GetSize(),
+				 .RawHash = IoHash::HashBuffer(CacheValue.second.GetData(), CacheValue.second.GetSize())},
+				{});
+		CacheEntries.insert({Key, CacheEntry{CacheValue.second, {}}});
+		return Key;
+	};
+
+	auto ValidateCacheEntry = [&](ZenCacheNamespace& Zcs,
+								  CidStore&			 CidStore,
+								  std::string_view	 Bucket,
+								  const IoHash&		 Key,
+								  bool				 ExpectCacheEntry,
+								  bool				 ExpectAttachments) {
+		const CacheEntry& Entry = CacheEntries[Key];
+		ZenCacheValue	  Value;
+		bool			  CacheExists = Zcs.Get(Bucket, Key, Value);
+		if (ExpectCacheEntry)
+		{
+			if (!CacheExists)
+			{
+				return false;
+			}
+			if (Value.Value.GetSize() != Entry.Data.GetSize())
+			{
+				return false;
+			}
+			if (!Value.Value.GetView().EqualBytes(Entry.Data.GetView()))
+			{
+				return false;
+			}
+		}
+		else if (CacheExists)
+		{
+			return false;
+		}
+
+		if (ExpectAttachments)
+		{
+			for (const auto& Attachment : Entry.Attachments)
+			{
+				IoHash	 AttachmentHash = Attachment.second.DecodeRawHash();
+				IoBuffer StoredData		= CidStore.FindChunkByCid(AttachmentHash);
+				if (!StoredData)
+				{
+					return false;
+				}
+				if (!StoredData.GetView().EqualBytes(Attachment.second.GetCompressed().Flatten().GetView()))
+				{
+					return false;
+				}
+			}
+		}
+		else
+		{
+			for (const auto& Attachment : Entry.Attachments)
+			{
+				IoHash AttachmentHash = Attachment.second.DecodeRawHash();
+				if (CidStore.ContainsChunk(AttachmentHash))
+				{
+					return false;
+				}
+			}
+		}
+		return true;
+	};
+
+	std::vector<IoHash> CacheRecords;
+	std::vector<IoHash> UnstructuredCacheValues;
+
+	const auto TearDrinkerBucket = "teardrinker"sv;
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		// Create some basic data
+		{
+			// Structured record with attachments
+			auto Attachments1 = CreateCompressedAttachment(CidStore, std::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87});
+			CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments1));
+
+			// Structured record with reuse of attachments
+			auto Attachments2 = CreateCompressedAttachment(CidStore, std::vector<size_t>{971});
+			Attachments2.push_back(Attachments1[0]);
+			Attachments2.push_back(Attachments1[1]);
+			CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments2));
+		}
+
+		CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, {}));
+
+		{
+			// Unstructured cache values
+			UnstructuredCacheValues.push_back(CreateCacheValue(Zcs, TearDrinkerBucket, 84));
+			UnstructuredCacheValues.push_back(CreateCacheValue(Zcs, TearDrinkerBucket, 591));
+			UnstructuredCacheValues.push_back(CreateCacheValue(Zcs, TearDrinkerBucket, 1024 * 1024 * 3 + 7));
+			UnstructuredCacheValues.push_back(CreateCacheValue(Zcs, TearDrinkerBucket, 71));
+		}
+	}
+
+	SUBCASE("expire nothing")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(1),
+													   .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
+													   .CollectSmallObjects	   = false,
+													   .IsDeleteMode		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(0u, Result.ExpiredItems);
+		CHECK_EQ(0u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(0u, Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all large objects, delete nothing")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = false,
+													   .IsDeleteMode		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(1u, Result.ExpiredItems);
+		CHECK_EQ(0u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(0u, Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all object, delete nothing")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(7u, Result.ExpiredItems);
+		CHECK_EQ(0u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(0u, Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all large objects, skip cid")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = false,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = true});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(1u, Result.ExpiredItems);
+		CHECK_EQ(1u, Result.DeletedItems);
+		CHECK_EQ(0u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all objects, skip cid")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = true});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(7u, Result.ExpiredItems);
+		CHECK_EQ(7u, Result.DeletedItems);
+		CHECK_EQ(0u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_GE(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], false, true));
+	}
+	SUBCASE("expire all large objects")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = false,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(1u, Result.ExpiredItems);	// Only one cache value is pruned/deleted as that is the only large item in the cache (all other
+											// large items as in cas)
+		CHECK_EQ(1u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u,
+				 Result.PrunedReferences);	// We won't remove any references since all referencers are small which retains all references
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDiskSpace);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], true, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+	SUBCASE("expire all objects")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		GcResult		  Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		= GcClock::Now() + std::chrono::minutes(1),
+																.ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+																.CollectSmallObjects	= true,
+																.IsDeleteMode			= true,
+																.SkipCidDelete			= false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(7u, Result.ExpiredItems);
+		CHECK_EQ(7u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(5u, Result.PrunedReferences);
+		CHECK_EQ(5u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, false));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], false, false));
+	}
+
+	SUBCASE("keep 1 cache record, skip cid")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = true});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(6u, Result.ExpiredItems);
+		CHECK_EQ(6u, Result.DeletedItems);
+		CHECK_EQ(0u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], false, false));
+	}
+
+	SUBCASE("keep 2 cache records")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(5u, Result.ExpiredItems);
+		CHECK_EQ(5u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, false));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], false, false));
+	}
+
+	SUBCASE("keep 3 cache value")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = false});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(4u, Result.ExpiredItems);
+		CHECK_EQ(4u, Result.DeletedItems);
+		CHECK_EQ(5u, Result.References);
+		CHECK_EQ(5u, Result.PrunedReferences);
+		CHECK_EQ(5u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		CHECK_EQ(0u, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, false));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+
+	SUBCASE("keep 3 cache value, skip cid")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		// Prime so we can check GC of memory layer
+		ZenCacheValue Dummy;
+		Zcs.Get(TearDrinkerBucket, CacheRecords[0], Dummy);
+		Zcs.Get(TearDrinkerBucket, CacheRecords[1], Dummy);
+		Zcs.Get(TearDrinkerBucket, CacheRecords[2], Dummy);
+		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[0], Dummy);
+		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[1], Dummy);
+		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[2], Dummy);
+		Zcs.Get(TearDrinkerBucket, UnstructuredCacheValues[3], Dummy);
+
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = true});
+		CHECK_EQ(7u, Result.Items);
+		CHECK_EQ(4u, Result.ExpiredItems);
+		CHECK_EQ(4u, Result.DeletedItems);
+		CHECK_EQ(0u, Result.References);
+		CHECK_EQ(0u, Result.PrunedReferences);
+		CHECK_EQ(0u, Result.CompactedReferences);
+		CHECK_GT(Result.RemovedDiskSpace, 0);
+		uint64_t MemoryClean = CacheEntries[CacheRecords[0]].Data.GetSize() + CacheEntries[CacheRecords[1]].Data.GetSize() +
+							   CacheEntries[CacheRecords[2]].Data.GetSize() + CacheEntries[UnstructuredCacheValues[0]].Data.GetSize();
+		CHECK_EQ(MemoryClean, Result.RemovedMemory);
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[2], false, true));
+
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[0], false, false));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[1], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[2], true, true));
+		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, UnstructuredCacheValues[3], true, true));
+	}
+
+	SUBCASE("leave write block")
+	{
+		GcManager Gc;
+		CidStore  CidStore(Gc);
+		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path() / "cache",
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+
+		auto Attachments =
+			CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
+		IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments);
+
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecord, GcClock::Now() - std::chrono::minutes(2));
+
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[1], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[2], GcClock::Now() + std::chrono::minutes(2));
+
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[0], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::minutes(2));
+		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::minutes(2));
+
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::minutes(1),
+													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::minutes(1),
+													   .CollectSmallObjects	   = true,
+													   .IsDeleteMode		   = true,
+													   .SkipCidDelete		   = false});
+		CHECK_EQ(8u, Result.Items);
+		CHECK_EQ(1u, Result.ExpiredItems);
+		CHECK_EQ(1u, Result.DeletedItems);
+		CHECK_EQ(9u, Result.References);
+		CHECK_EQ(4u, Result.PrunedReferences);
+		CHECK_EQ(4u, Result.CompactedReferences);
+		CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(),
+				 Result.RemovedDiskSpace);
+		uint64_t MemoryClean = CacheEntries[CacheRecord].Data.GetSize();
+		CHECK_EQ(MemoryClean, Result.RemovedMemory);
+	}
+}
+
 #endif
 
 void
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index dacf482d8..a3cac0d44 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -106,6 +106,10 @@ public:
 
 	CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const;
 
+#if ZEN_WITH_TESTS
+	void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time);
+#endif	// ZEN_WITH_TESTS
+
 private:
 	GcManager&			  m_Gc;
 	JobQueue&			  m_JobQueue;