remove reference caching (#658)

* remove reference caching

1 files changed, 101 insertions, 503 deletions

diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 615f8640f..7274626ee 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -957,11 +957,6 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const
 
 	m_AccessTimes.resize(EntryCount, AccessTime(GcClock::TickCount()));
 
-	if (m_Configuration.EnableReferenceCaching)
-	{
-		m_FirstReferenceIndex.resize(EntryCount);
-	}
-
 	OutVersion = CacheBucketIndexHeader::Version2;
 	return Header.LogPosition;
 }
@@ -1023,11 +1018,6 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std::
 
 	m_AccessTimes.resize(m_Payloads.size(), AccessTime(GcClock::TickCount()));
 
-	if (m_Configuration.EnableReferenceCaching)
-	{
-		m_FirstReferenceIndex.resize(m_Payloads.size());
-	}
-
 	if (InvalidEntryCount)
 	{
 		ZEN_WARN("found {} invalid entries in '{}'", InvalidEntryCount, m_BucketDir);
@@ -1050,10 +1040,6 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco
 	m_FreeMetaDatas.clear();
 	m_MemCachedPayloads.clear();
 	m_FreeMemCachedPayloads.clear();
-	m_FirstReferenceIndex.clear();
-	m_ReferenceHashes.clear();
-	m_NextReferenceHashesIndexes.clear();
-	m_ReferenceCount = 0;
 
 	std::filesystem::path LogPath	= GetLogPath(m_BucketDir, m_BucketName);
 	std::filesystem::path IndexPath = GetIndexPath(m_BucketDir, m_BucketName);
@@ -1419,10 +1405,6 @@ ZenCacheDiskLayer::CacheBucket::Drop()
 	m_FreeMetaDatas.clear();
 	m_MemCachedPayloads.clear();
 	m_FreeMemCachedPayloads.clear();
-	m_FirstReferenceIndex.clear();
-	m_ReferenceHashes.clear();
-	m_NextReferenceHashesIndexes.clear();
-	m_ReferenceCount = 0;
 	m_StandaloneSize.store(0);
 	m_OuterCacheMemoryUsage.fetch_sub(m_MemCachedSize.load());
 	m_MemCachedSize.store(0);
@@ -1738,13 +1720,9 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 				for (const IoHash& BadKey : BadKeys)
 				{
 					// Log a tombstone and delete the in-memory index for the bad entry
-					const auto	   It	   = m_Index.find(BadKey);
-					BucketPayload& Payload = m_Payloads[It->second];
-					if (m_Configuration.EnableReferenceCaching)
-					{
-						RemoveReferences(IndexLock, m_FirstReferenceIndex[It->second]);
-					}
-					DiskLocation Location = Payload.Location;
+					const auto	   It		= m_Index.find(BadKey);
+					BucketPayload& Payload	= m_Payloads[It->second];
+					DiskLocation   Location = Payload.Location;
 					if (Location.IsFlagSet(DiskLocation::kStandaloneFile))
 					{
 						m_StandaloneSize.fetch_sub(Location.Size(), std::memory_order::relaxed);
@@ -1782,12 +1760,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 				std::vector<AccessTime>		AccessTimes;
 				std::vector<BucketMetaData> MetaDatas;
 				std::vector<MemCacheData>	MemCachedPayloads;
-				std::vector<ReferenceIndex> FirstReferenceIndex;
 				IndexMap					Index;
 
 				{
 					RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-					CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+					CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index);
 				}
 			}
 		}
@@ -1835,10 +1812,9 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 
 	const GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count();
 
-	IndexMap					Index;
-	std::vector<AccessTime>		AccessTimes;
-	std::vector<BucketPayload>	Payloads;
-	std::vector<ReferenceIndex> FirstReferenceIndex;
+	IndexMap				   Index;
+	std::vector<AccessTime>	   AccessTimes;
+	std::vector<BucketPayload> Payloads;
 	{
 		RwLock::SharedLockScope __(m_IndexLock);
 #if CALCULATE_BLOCKING_TIME
@@ -1853,10 +1829,9 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 		{
 			return;
 		}
-		Index				= m_Index;
-		AccessTimes			= m_AccessTimes;
-		Payloads			= m_Payloads;
-		FirstReferenceIndex = m_FirstReferenceIndex;
+		Index		= m_Index;
+		AccessTimes = m_AccessTimes;
+		Payloads	= m_Payloads;
 	}
 
 	std::vector<IoHash> ExpiredKeys;
@@ -1893,40 +1868,6 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 		{
 			continue;
 		}
-		if (m_Configuration.EnableReferenceCaching)
-		{
-			if (FirstReferenceIndex.empty() || FirstReferenceIndex[PayloadIndex] == ReferenceIndex::Unknown())
-			{
-				StructuredItemsWithUnknownAttachments.push_back(Entry);
-				continue;
-			}
-
-			bool ReferencesAreKnown = false;
-			{
-				RwLock::SharedLockScope IndexLock(m_IndexLock);
-#if CALCULATE_BLOCKING_TIME
-				Stopwatch  Timer;
-				const auto ___ = MakeGuard([&] {
-					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
-					WriteBlockTimeUs += ElapsedUs;
-					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
-				});
-#endif	// CALCULATE_BLOCKING_TIME
-				if (auto It = m_Index.find(Key); It != m_Index.end())
-				{
-					ReferencesAreKnown = GetReferences(IndexLock, m_FirstReferenceIndex[It->second], Cids);
-				}
-			}
-			if (ReferencesAreKnown)
-			{
-				if (Cids.size() >= 1024)
-				{
-					GcCtx.AddRetainedCids(Cids);
-					Cids.clear();
-				}
-				continue;
-			}
-		}
 		StructuredItemsWithUnknownAttachments.push_back(Entry);
 	}
 
@@ -1980,34 +1921,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 			ZEN_ASSERT(Buffer);
 			ZEN_ASSERT(Buffer.GetContentType() == ZenContentType::kCbObject);
 			CbObjectView Obj(Buffer.GetData());
-			size_t		 CurrentCidCount = Cids.size();
 			Obj.IterateAttachments([&Cids](CbFieldView Field) { Cids.push_back(Field.AsAttachment()); });
-			if (m_Configuration.EnableReferenceCaching)
-			{
-				RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-#if CALCULATE_BLOCKING_TIME
-				Stopwatch  Timer;
-				const auto ___ = MakeGuard([&] {
-					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
-					ReadBlockTimeUs += ElapsedUs;
-					ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
-				});
-#endif	// CALCULATE_BLOCKING_TIME
-				if (auto It = m_Index.find(Key); It != m_Index.end())
-				{
-					if (m_FirstReferenceIndex[It->second] == ReferenceIndex::Unknown())
-					{
-						SetReferences(IndexLock,
-									  m_FirstReferenceIndex[It->second],
-									  std::span<IoHash>(Cids.data() + CurrentCidCount, Cids.size() - CurrentCidCount));
-					}
-					else
-					{
-						Cids.resize(CurrentCidCount);
-						(void)GetReferences(IndexLock, m_FirstReferenceIndex[It->second], Cids);
-					}
-				}
-			}
 			if (Cids.size() >= 1024)
 			{
 				GcCtx.AddRetainedCids(Cids);
@@ -2083,7 +1997,6 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			std::vector<AccessTime>		AccessTimes;
 			std::vector<BucketMetaData> MetaDatas;
 			std::vector<MemCacheData>	MemCachedPayloads;
-			std::vector<ReferenceIndex> FirstReferenceIndex;
 			IndexMap					Index;
 			{
 				RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
@@ -2093,7 +2006,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 					WriteBlockTimeUs += ElapsedUs;
 					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
 				});
-				CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+				CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index);
 			}
 			GcCtx.AddDeletedCids(std::vector<IoHash>(DeletedChunks.begin(), DeletedChunks.end()));
 		}
@@ -2549,9 +2462,13 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 
 	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 	ValueLock.ReleaseNow();
-	if (m_UpdatedKeys)
+	if (m_TrackedCacheKeys)
 	{
-		m_UpdatedKeys->insert(HashKey);
+		m_TrackedCacheKeys->insert(HashKey);
+	}
+	if (m_TrackedReferences)
+	{
+		m_TrackedReferences->insert(References.begin(), References.end());
 	}
 
 	PayloadIndex EntryIndex = {};
@@ -2561,24 +2478,15 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 		EntryIndex = PayloadIndex(m_Payloads.size());
 		m_Payloads.emplace_back(BucketPayload{.Location = Loc});
 		m_AccessTimes.emplace_back(GcClock::TickCount());
-		if (m_Configuration.EnableReferenceCaching)
-		{
-			m_FirstReferenceIndex.emplace_back(ReferenceIndex{});
-			SetReferences(IndexLock, m_FirstReferenceIndex.back(), References);
-		}
 		m_Index.insert_or_assign(HashKey, EntryIndex);
 	}
 	else
 	{
 		EntryIndex = It.value();
 		ZEN_ASSERT_SLOW(EntryIndex < PayloadIndex(m_AccessTimes.size()));
-		BucketPayload& Payload = m_Payloads[EntryIndex];
-		uint64_t	   OldSize = Payload.Location.Size();
-		Payload				   = BucketPayload{.Location = Loc};
-		if (m_Configuration.EnableReferenceCaching)
-		{
-			SetReferences(IndexLock, m_FirstReferenceIndex[EntryIndex], References);
-		}
+		BucketPayload& Payload	  = m_Payloads[EntryIndex];
+		uint64_t	   OldSize	  = Payload.Location.Size();
+		Payload					  = BucketPayload{.Location = Loc};
 		m_AccessTimes[EntryIndex] = GcClock::TickCount();
 		RemoveMemCachedData(IndexLock, Payload);
 		m_StandaloneSize.fetch_sub(OldSize, std::memory_order::relaxed);
@@ -2706,9 +2614,13 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 								m_SlogFile.Append({.Key = HashKey, .Location = Location});
 
 								RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-								if (m_UpdatedKeys)
+								if (m_TrackedCacheKeys)
+								{
+									m_TrackedCacheKeys->insert(HashKey);
+								}
+								if (m_TrackedReferences)
 								{
-									m_UpdatedKeys->insert(HashKey);
+									m_TrackedReferences->insert(References.begin(), References.end());
 								}
 								if (auto It = m_Index.find(HashKey); It != m_Index.end())
 								{
@@ -2721,22 +2633,12 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 
 									Payload					  = (BucketPayload{.Location = Location});
 									m_AccessTimes[EntryIndex] = GcClock::TickCount();
-
-									if (m_Configuration.EnableReferenceCaching)
-									{
-										SetReferences(IndexLock, m_FirstReferenceIndex[EntryIndex], References);
-									}
 								}
 								else
 								{
 									PayloadIndex EntryIndex = PayloadIndex(m_Payloads.size());
 									m_Payloads.emplace_back(BucketPayload{.Location = Location});
 									m_AccessTimes.emplace_back(GcClock::TickCount());
-									if (m_Configuration.EnableReferenceCaching)
-									{
-										m_FirstReferenceIndex.emplace_back(ReferenceIndex{});
-										SetReferences(IndexLock, m_FirstReferenceIndex.back(), References);
-									}
 									m_Index.insert_or_assign(HashKey, EntryIndex);
 								}
 							});
@@ -2750,8 +2652,12 @@ ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&)
 
 class DiskBucketStoreCompactor : public GcStoreCompactor
 {
+	using PayloadIndex	= ZenCacheDiskLayer::CacheBucket::PayloadIndex;
+	using BucketPayload = ZenCacheDiskLayer::CacheBucket::BucketPayload;
+	using CacheBucket	= ZenCacheDiskLayer::CacheBucket;
+
 public:
-	DiskBucketStoreCompactor(ZenCacheDiskLayer::CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
+	DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
 	: m_Bucket(Bucket)
 	, m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys))
 	{
@@ -2854,8 +2760,8 @@ public:
 
 		if (Ctx.Settings.CollectSmallObjects)
 		{
-			m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_UpdatedKeys = std::make_unique<HashSet>(); });
-			auto __ = MakeGuard([&]() { m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_UpdatedKeys.reset(); }); });
+			m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_TrackedCacheKeys = std::make_unique<HashSet>(); });
+			auto __ = MakeGuard([&]() { m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_TrackedCacheKeys.reset(); }); });
 
 			size_t					  InlineEntryCount = 0;
 			BlockStore::BlockUsageMap BlockUsage;
@@ -2863,9 +2769,9 @@ public:
 				RwLock::SharedLockScope ___(m_Bucket.m_IndexLock);
 				for (const auto& Entry : m_Bucket.m_Index)
 				{
-					ZenCacheDiskLayer::CacheBucket::PayloadIndex		 Index	 = Entry.second;
-					const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = m_Bucket.m_Payloads[Index];
-					const DiskLocation&									 Loc	 = Payload.Location;
+					PayloadIndex		 Index	 = Entry.second;
+					const BucketPayload& Payload = m_Bucket.m_Payloads[Index];
+					const DiskLocation&	 Loc	 = Payload.Location;
 
 					if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 					{
@@ -2901,9 +2807,9 @@ public:
 						RwLock::SharedLockScope ___(m_Bucket.m_IndexLock);
 						for (const auto& Entry : m_Bucket.m_Index)
 						{
-							ZenCacheDiskLayer::CacheBucket::PayloadIndex		 Index	 = Entry.second;
-							const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = m_Bucket.m_Payloads[Index];
-							const DiskLocation&									 Loc	 = Payload.Location;
+							PayloadIndex		 Index	 = Entry.second;
+							const BucketPayload& Payload = m_Bucket.m_Payloads[Index];
+							const DiskLocation&	 Loc	 = Payload.Location;
 
 							if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 							{
@@ -2938,18 +2844,19 @@ public:
 									size_t		  ChunkIndex = Moved.first;
 									const IoHash& Key		 = BlockCompactStateKeys[ChunkIndex];
 
-									if (m_Bucket.m_UpdatedKeys->contains(Key))
+									ZEN_ASSERT(m_Bucket.m_TrackedCacheKeys);
+									if (m_Bucket.m_TrackedCacheKeys->contains(Key))
 									{
 										continue;
 									}
 
 									if (auto It = m_Bucket.m_Index.find(Key); It != m_Bucket.m_Index.end())
 									{
-										ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload	   = m_Bucket.m_Payloads[It->second];
-										const BlockStoreLocation&					   NewLocation = Moved.second;
-										Payload.Location										   = DiskLocation(NewLocation,
-																		  m_Bucket.m_Configuration.PayloadAlignment,
-																		  Payload.Location.GetFlags());
+										BucketPayload&			  Payload	  = m_Bucket.m_Payloads[It->second];
+										const BlockStoreLocation& NewLocation = Moved.second;
+										Payload.Location					  = DiskLocation(NewLocation,
+																		 m_Bucket.m_Configuration.PayloadAlignment,
+																		 Payload.Location.GetFlags());
 										MovedEntries.push_back({.Key = Key, .Location = Payload.Location});
 									}
 								}
@@ -2991,17 +2898,31 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
-		if (!Ctx.Settings.Verbose)
+		if (Ctx.Settings.Verbose)
+		{
+			ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}",
+					 m_BucketDir,
+					 Stats.CheckedCount,
+					 Stats.FoundCount,
+					 Stats.DeletedCount,
+					 NiceBytes(Stats.FreedMemory),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		}
+		bool Expected = false;
+		if (m_IsFlushing || !m_IsFlushing.compare_exchange_strong(Expected, true))
 		{
 			return;
 		}
-		ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}",
-				 m_BucketDir,
-				 Stats.CheckedCount,
-				 Stats.FoundCount,
-				 Stats.DeletedCount,
-				 NiceBytes(Stats.FreedMemory),
-				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		auto FlushingGuard = MakeGuard([&] { m_IsFlushing.store(false); });
+
+		try
+		{
+			SaveSnapshot([]() { return 0; });
+		}
+		catch (std::exception& Ex)
+		{
+			ZEN_WARN("Failed to write index and manifest after RemoveExpiredData in '{}'. Reason: '{}'", m_BucketDir, Ex.what());
+		}
 	});
 
 	const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
@@ -3080,11 +3001,10 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 		std::vector<AccessTime>		AccessTimes;
 		std::vector<BucketMetaData> MetaDatas;
 		std::vector<MemCacheData>	MemCachedPayloads;
-		std::vector<ReferenceIndex> FirstReferenceIndex;
 		IndexMap					Index;
 		{
 			RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-			CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+			CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index);
 		}
 	}
 
@@ -3098,10 +3018,9 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 
 class DiskBucketReferenceChecker : public GcReferenceChecker
 {
-	using PayloadIndex	 = ZenCacheDiskLayer::CacheBucket::PayloadIndex;
-	using BucketPayload	 = ZenCacheDiskLayer::CacheBucket::BucketPayload;
-	using CacheBucket	 = ZenCacheDiskLayer::CacheBucket;
-	using ReferenceIndex = ZenCacheDiskLayer::CacheBucket::ReferenceIndex;
+	using PayloadIndex	= ZenCacheDiskLayer::CacheBucket::PayloadIndex;
+	using BucketPayload = ZenCacheDiskLayer::CacheBucket::BucketPayload;
+	using CacheBucket	= ZenCacheDiskLayer::CacheBucket;
 
 public:
 	DiskBucketReferenceChecker(CacheBucket& Owner) : m_CacheBucket(Owner) {}
@@ -3111,12 +3030,7 @@ public:
 		try
 		{
 			m_IndexLock.reset();
-			if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
-			{
-				m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
-				// If reference caching is not enabled, we temporarily used the data structure for reference caching, lets reset it
-				m_CacheBucket.ClearReferenceCache();
-			}
+			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
 		}
 		catch (std::exception& Ex)
 		{
@@ -3136,28 +3050,18 @@ public:
 			}
 			ZEN_INFO("GCV2: cachebucket [PRECACHE] '{}': found {} references in {}",
 					 m_CacheBucket.m_BucketDir,
-					 m_CacheBucket.m_ReferenceCount,
+					 m_PrecachedReferences.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		std::vector<IoHash> UpdateKeys;
-		std::vector<size_t> ReferenceCounts;
-		std::vector<IoHash> References;
-
-		auto GetAttachments = [&References, &ReferenceCounts](const void* CbObjectData) {
-			size_t		 CurrentReferenceCount = References.size();
+		auto GetAttachments = [&](const void* CbObjectData) {
 			CbObjectView Obj(CbObjectData);
-			Obj.IterateAttachments([&References](CbFieldView Field) { References.emplace_back(Field.AsAttachment()); });
-			ReferenceCounts.push_back(References.size() - CurrentReferenceCount);
+			Obj.IterateAttachments([&](CbFieldView Field) { m_PrecachedReferences.emplace_back(Field.AsAttachment()); });
 		};
 
 		// Refresh cache
 		{
-			// If reference caching is enabled the references will be updated at modification for us so we don't need to track modifications
-			if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
-			{
-				m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys = std::make_unique<HashSet>(); });
-			}
+			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<HashSet>(); });
 
 			std::vector<IoHash> StandaloneKeys;
 			{
@@ -3170,7 +3074,6 @@ public:
 					uint32_t Size;
 				};
 				std::vector<std::vector<InlineEntry>> EntriesPerBlock;
-				size_t								  UpdateCount = 0;
 				{
 					RwLock::SharedLockScope IndexLock(m_CacheBucket.m_IndexLock);
 					for (const auto& Entry : m_CacheBucket.m_Index)
@@ -3178,7 +3081,7 @@ public:
 						if (Ctx.IsCancelledFlag.load())
 						{
 							IndexLock.ReleaseNow();
-							m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
+							m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
 							return;
 						}
 
@@ -3190,12 +3093,6 @@ public:
 						{
 							continue;
 						}
-						if (m_CacheBucket.m_Configuration.EnableReferenceCaching &&
-							m_CacheBucket.m_FirstReferenceIndex[EntryIndex] != ReferenceIndex::Unknown())
-						{
-							continue;
-						}
-						UpdateCount++;
 						const IoHash& Key = Entry.first;
 						if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 						{
@@ -3222,8 +3119,6 @@ public:
 					}
 				}
 
-				UpdateKeys.reserve(UpdateCount);
-
 				for (auto It : BlockIndexToEntriesPerBlockIndex)
 				{
 					uint32_t BlockIndex = It.first;
@@ -3242,11 +3137,7 @@ public:
 						BasicFileBuffer BlockBuffer(BlockFile->GetBasicFile(), 32768);
 						for (const InlineEntry& InlineEntry : InlineEntries)
 						{
-							if ((InlineEntry.Offset + InlineEntry.Size) > BlockFileSize)
-							{
-								ReferenceCounts.push_back(0);
-							}
-							else
+							if ((InlineEntry.Offset + InlineEntry.Size) <= BlockFileSize)
 							{
 								MemoryView ChunkView = BlockBuffer.MakeView(InlineEntry.Size, InlineEntry.Offset);
 								if (ChunkView.GetSize() == InlineEntry.Size)
@@ -3255,92 +3146,29 @@ public:
 								}
 								else
 								{
-									std::vector<uint8_t> Buffer(InlineEntry.Size);
-									BlockBuffer.Read(Buffer.data(), InlineEntry.Size, InlineEntry.Offset);
-									GetAttachments(Buffer.data());
+									IoBuffer Buffer = BlockFile->GetChunk(InlineEntry.Offset, InlineEntry.Size);
+									GetAttachments(Buffer.GetData());
 								}
 							}
-							const IoHash& Key = InlineKeys[InlineEntry.InlineKeyIndex];
-							UpdateKeys.push_back(Key);
 						}
 					}
 				}
 			}
+			for (const IoHash& Key : StandaloneKeys)
 			{
-				for (const IoHash& Key : StandaloneKeys)
+				if (Ctx.IsCancelledFlag.load())
 				{
-					if (Ctx.IsCancelledFlag.load())
-					{
-						m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
-						return;
-					}
-
-					IoBuffer Buffer = m_CacheBucket.GetStandaloneCacheValue(ZenContentType::kCbObject, Key);
-					if (!Buffer)
-					{
-						continue;
-					}
-
-					GetAttachments(Buffer.GetData());
-					UpdateKeys.push_back(Key);
+					m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
+					return;
 				}
-			}
-		}
-
-		{
-			size_t					   ReferenceOffset = 0;
-			RwLock::ExclusiveLockScope IndexLock(m_CacheBucket.m_IndexLock);
 
-			if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
-			{
-				ZEN_ASSERT(m_CacheBucket.m_FirstReferenceIndex.empty());
-				ZEN_ASSERT(m_CacheBucket.m_ReferenceHashes.empty());
-				ZEN_ASSERT(m_CacheBucket.m_NextReferenceHashesIndexes.empty());
-				ZEN_ASSERT(m_CacheBucket.m_ReferenceCount == 0);
-				ZEN_ASSERT(m_CacheBucket.m_UpdatedKeys);
-
-				// If reference caching is not enabled, we will resize and use the data structure in place for reference caching when
-				// we figure out what this bucket references. This will be reset once the DiskBucketReferenceChecker is deleted.
-				m_CacheBucket.m_FirstReferenceIndex.resize(m_CacheBucket.m_Payloads.size(), ReferenceIndex::Unknown());
-				m_CacheBucket.m_ReferenceHashes.reserve(References.size());
-				m_CacheBucket.m_NextReferenceHashesIndexes.reserve(References.size());
-			}
-			else
-			{
-				ZEN_ASSERT(!m_CacheBucket.m_UpdatedKeys);
-			}
-
-			for (size_t Index = 0; Index < UpdateKeys.size(); Index++)
-			{
-				const IoHash& Key			 = UpdateKeys[Index];
-				size_t		  ReferenceCount = ReferenceCounts[Index];
-				if (auto It = m_CacheBucket.m_Index.find(Key); It != m_CacheBucket.m_Index.end())
+				IoBuffer Buffer = m_CacheBucket.GetStandaloneCacheValue(ZenContentType::kCbObject, Key);
+				if (!Buffer)
 				{
-					PayloadIndex EntryIndex = It->second;
-					if (m_CacheBucket.m_Configuration.EnableReferenceCaching)
-					{
-						if (m_CacheBucket.m_FirstReferenceIndex[EntryIndex] != ReferenceIndex::Unknown())
-						{
-							// The reference data is valid and what we have is old/redundant
-							continue;
-						}
-					}
-					else if (m_CacheBucket.m_UpdatedKeys->contains(Key))
-					{
-						// Our pre-cache data is invalid
-						continue;
-					}
-
-					m_CacheBucket.SetReferences(IndexLock,
-												m_CacheBucket.m_FirstReferenceIndex[EntryIndex],
-												std::span<IoHash>{References.data() + ReferenceOffset, ReferenceCount});
+					continue;
 				}
-				ReferenceOffset += ReferenceCount;
-			}
 
-			if (m_CacheBucket.m_Configuration.EnableReferenceCaching && !UpdateKeys.empty())
-			{
-				m_CacheBucket.CompactReferences(IndexLock);
+				GetAttachments(Buffer.GetData());
 			}
 		}
 	}
@@ -3357,65 +3185,25 @@ public:
 			}
 			ZEN_INFO("GCV2: cachebucket [LOCKSTATE] '{}': found {} references in {}",
 					 m_CacheBucket.m_BucketDir,
-					 m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
+					 m_PrecachedReferences.size() + m_UncachedReferences.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		m_IndexLock = std::make_unique<RwLock::SharedLockScope>(m_CacheBucket.m_IndexLock);
 		if (Ctx.IsCancelledFlag.load())
 		{
-			m_UncachedReferences.clear();
+			m_UncachedReferences = {};
 			m_IndexLock.reset();
-			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
+			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
 			return;
 		}
 
-		if (m_CacheBucket.m_UpdatedKeys)
-		{
-			const HashSet& UpdatedKeys(*m_CacheBucket.m_UpdatedKeys);
-			for (const IoHash& Key : UpdatedKeys)
-			{
-				if (Ctx.IsCancelledFlag.load())
-				{
-					m_UncachedReferences.clear();
-					m_IndexLock.reset();
-					m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
-					return;
-				}
+		ZEN_ASSERT(m_CacheBucket.m_TrackedReferences);
 
-				auto It = m_CacheBucket.m_Index.find(Key);
-				if (It == m_CacheBucket.m_Index.end())
-				{
-					continue;
-				}
-
-				PayloadIndex		 EntryIndex = It->second;
-				const BucketPayload& Payload	= m_CacheBucket.m_Payloads[EntryIndex];
-				const DiskLocation&	 Loc		= Payload.Location;
-
-				if (!Loc.IsFlagSet(DiskLocation::kStructured))
-				{
-					continue;
-				}
-
-				IoBuffer Buffer;
-				if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
-				{
-					Buffer = m_CacheBucket.GetStandaloneCacheValue(Loc.GetContentType(), Key);
-				}
-				else
-				{
-					Buffer = m_CacheBucket.GetInlineCacheValue(Loc);
-				}
-
-				if (Buffer)
-				{
-					ZEN_ASSERT(Buffer.GetContentType() == ZenContentType::kCbObject);
-					CbObjectView Obj(Buffer.GetData());
-					Obj.IterateAttachments([this](CbFieldView Field) { m_UncachedReferences.insert(Field.AsAttachment()); });
-				}
-			}
-		}
+		HashSet& AddedReferences(*m_CacheBucket.m_TrackedReferences);
+		m_UncachedReferences.reserve(AddedReferences.size());
+		m_UncachedReferences.insert(m_UncachedReferences.end(), AddedReferences.begin(), AddedReferences.end());
+		AddedReferences = {};
 	}
 
 	virtual void RemoveUsedReferencesFromSet(GcCtx& Ctx, HashSet& IoCids) override
@@ -3437,7 +3225,7 @@ public:
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		for (const IoHash& ReferenceHash : m_CacheBucket.m_ReferenceHashes)
+		for (const IoHash& ReferenceHash : m_PrecachedReferences)
 		{
 			if (IoCids.erase(ReferenceHash) == 1)
 			{
@@ -3461,7 +3249,8 @@ public:
 	}
 	CacheBucket&							 m_CacheBucket;
 	std::unique_ptr<RwLock::SharedLockScope> m_IndexLock;
-	HashSet									 m_UncachedReferences;
+	std::vector<IoHash>						 m_PrecachedReferences;
+	std::vector<IoHash>						 m_UncachedReferences;
 };
 
 std::vector<GcReferenceChecker*>
@@ -3490,200 +3279,18 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::CompactReferences(RwLock::ExclusiveLockScope&)
-{
-	ZEN_TRACE_CPU("Z$::Bucket::CompactReferences");
-
-	std::vector<ReferenceIndex> FirstReferenceIndex;
-	std::vector<IoHash>			NewReferenceHashes;
-	std::vector<ReferenceIndex> NewNextReferenceHashesIndexes;
-
-	FirstReferenceIndex.reserve(m_ReferenceCount);
-	NewReferenceHashes.reserve(m_ReferenceCount);
-	NewNextReferenceHashesIndexes.reserve(m_ReferenceCount);
-
-	for (const auto& It : m_Index)
-	{
-		ReferenceIndex SourceIndex = m_FirstReferenceIndex[It.second];
-		if (SourceIndex == ReferenceIndex::Unknown())
-		{
-			FirstReferenceIndex.push_back(ReferenceIndex{});
-			continue;
-		}
-		if (SourceIndex == ReferenceIndex::None())
-		{
-			FirstReferenceIndex.push_back(ReferenceIndex::None());
-			continue;
-		}
-		FirstReferenceIndex.push_back(ReferenceIndex{NewNextReferenceHashesIndexes.size()});
-		NewReferenceHashes.push_back(m_ReferenceHashes[SourceIndex]);
-		NewNextReferenceHashesIndexes.push_back(ReferenceIndex::None());
-
-		SourceIndex = m_NextReferenceHashesIndexes[SourceIndex];
-		while (SourceIndex != ReferenceIndex::None())
-		{
-			NewNextReferenceHashesIndexes.back() = ReferenceIndex{NewReferenceHashes.size()};
-			NewReferenceHashes.push_back(m_ReferenceHashes[SourceIndex]);
-			NewNextReferenceHashesIndexes.push_back(ReferenceIndex::None());
-			SourceIndex = m_NextReferenceHashesIndexes[SourceIndex];
-		}
-	}
-	m_FirstReferenceIndex.swap(FirstReferenceIndex);
-	m_ReferenceHashes.swap(NewReferenceHashes);
-	m_ReferenceHashes.shrink_to_fit();
-	m_NextReferenceHashesIndexes.swap(NewNextReferenceHashesIndexes);
-	m_NextReferenceHashesIndexes.shrink_to_fit();
-	m_ReferenceCount = m_ReferenceHashes.size();
-}
-
-ZenCacheDiskLayer::CacheBucket::ReferenceIndex
-ZenCacheDiskLayer::CacheBucket::AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key)
-{
-	ReferenceIndex NewIndex = ReferenceIndex{m_ReferenceHashes.size()};
-	m_ReferenceHashes.push_back(Key);
-	m_NextReferenceHashesIndexes.emplace_back(ReferenceIndex::None());
-	m_ReferenceCount++;
-	return NewIndex;
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::SetReferences(RwLock::ExclusiveLockScope& Lock,
-											  ReferenceIndex&			  FirstReferenceIndex,
-											  std::span<IoHash>			  References)
-{
-	auto ReferenceIt = References.begin();
-
-	if (FirstReferenceIndex == ReferenceIndex::Unknown())
-	{
-		FirstReferenceIndex = ReferenceIndex::None();
-	}
-
-	ReferenceIndex CurrentIndex = FirstReferenceIndex;
-	if (CurrentIndex != ReferenceIndex::None())
-	{
-		if (ReferenceIt != References.end())
-		{
-			ZEN_ASSERT_SLOW(*ReferenceIt != IoHash::Zero);
-			if (CurrentIndex == ReferenceIndex::None())
-			{
-				CurrentIndex		= AllocateReferenceEntry(Lock, *ReferenceIt);
-				FirstReferenceIndex = CurrentIndex;
-			}
-			else
-			{
-				m_ReferenceHashes[CurrentIndex] = *ReferenceIt;
-			}
-			ReferenceIt++;
-		}
-	}
-	else
-	{
-		if (ReferenceIt != References.end())
-		{
-			ZEN_ASSERT_SLOW(*ReferenceIt != IoHash::Zero);
-			CurrentIndex = AllocateReferenceEntry(Lock, *ReferenceIt);
-			ReferenceIt++;
-		}
-		FirstReferenceIndex = CurrentIndex;
-	}
-
-	while (ReferenceIt != References.end())
-	{
-		ZEN_ASSERT(CurrentIndex != ReferenceIndex::None());
-		ZEN_ASSERT_SLOW(*ReferenceIt != IoHash::Zero);
-		ReferenceIndex NextReferenceIndex = m_NextReferenceHashesIndexes[CurrentIndex];
-		if (NextReferenceIndex == ReferenceIndex::None())
-		{
-			NextReferenceIndex						   = AllocateReferenceEntry(Lock, *ReferenceIt);
-			m_NextReferenceHashesIndexes[CurrentIndex] = NextReferenceIndex;
-		}
-		else
-		{
-			m_ReferenceHashes[NextReferenceIndex] = *ReferenceIt;
-		}
-		CurrentIndex = NextReferenceIndex;
-		ReferenceIt++;
-	}
-
-	while (CurrentIndex != ReferenceIndex::None())
-	{
-		ReferenceIndex NextIndex = m_NextReferenceHashesIndexes[CurrentIndex];
-		if (NextIndex != ReferenceIndex::None())
-		{
-			m_ReferenceHashes[CurrentIndex] = IoHash::Zero;
-			ZEN_ASSERT(m_ReferenceCount > 0);
-			m_ReferenceCount--;
-			m_NextReferenceHashesIndexes[CurrentIndex] = ReferenceIndex::None();
-		}
-		CurrentIndex = NextIndex;
-	}
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::RemoveReferences(RwLock::ExclusiveLockScope&, ReferenceIndex& FirstReferenceIndex)
-{
-	if (FirstReferenceIndex == ReferenceIndex::Unknown())
-	{
-		return;
-	}
-	ReferenceIndex CurrentIndex = FirstReferenceIndex;
-	while (CurrentIndex == ReferenceIndex::None())
-	{
-		m_ReferenceHashes[CurrentIndex] = IoHash::Zero;
-		ZEN_ASSERT(m_ReferenceCount > 0);
-		m_ReferenceCount--;
-		CurrentIndex = m_NextReferenceHashesIndexes[CurrentIndex];
-	}
-	FirstReferenceIndex = {};
-}
-
-bool
-ZenCacheDiskLayer::CacheBucket::LockedGetReferences(ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
-{
-	if (FirstReferenceIndex == ReferenceIndex::Unknown())
-	{
-		return false;
-	}
-
-	ReferenceIndex CurrentIndex = FirstReferenceIndex;
-	while (CurrentIndex != ReferenceIndex::None())
-	{
-		ZEN_ASSERT_SLOW(m_ReferenceHashes[CurrentIndex] != IoHash::Zero);
-		OutReferences.push_back(m_ReferenceHashes[CurrentIndex]);
-		CurrentIndex = m_NextReferenceHashesIndexes[CurrentIndex];
-	}
-	return true;
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::ClearReferenceCache()
-{
-	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-	Reset(m_FirstReferenceIndex);
-	Reset(m_ReferenceHashes);
-	Reset(m_NextReferenceHashesIndexes);
-	m_ReferenceCount = 0;
-}
-
-void
 ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&,
 											 std::vector<BucketPayload>&  Payloads,
 											 std::vector<AccessTime>&	  AccessTimes,
 											 std::vector<BucketMetaData>& MetaDatas,
 											 std::vector<MemCacheData>&	  MemCachedPayloads,
-											 std::vector<ReferenceIndex>& FirstReferenceIndex,
-											 IndexMap&					  Index,
-											 RwLock::ExclusiveLockScope&  IndexLock)
+											 IndexMap&					  Index)
 {
 	ZEN_TRACE_CPU("Z$::Bucket::CompactState");
 
 	size_t EntryCount = m_Index.size();
 	Payloads.reserve(EntryCount);
 	AccessTimes.reserve(EntryCount);
-	if (m_Configuration.EnableReferenceCaching)
-	{
-		FirstReferenceIndex.reserve(EntryCount);
-	}
 	Index.reserve(EntryCount);
 	Index.min_load_factor(IndexMinLoadFactor);
 	Index.max_load_factor(IndexMaxLoadFactor);
@@ -3704,10 +3311,6 @@ ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&,
 				MemCacheData{.Payload = std::move(m_MemCachedPayloads[Payload.MemCached].Payload), .OwnerIndex = EntryIndex});
 			Payload.MemCached = MemCachedIndex(gsl::narrow<uint32_t>(MemCachedPayloads.size() - 1));
 		}
-		if (m_Configuration.EnableReferenceCaching)
-		{
-			FirstReferenceIndex.push_back(m_FirstReferenceIndex[It.second]);
-		}
 		Index.insert({It.first, EntryIndex});
 	}
 	m_Index.swap(Index);
@@ -3717,11 +3320,6 @@ ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&,
 	Reset(m_FreeMetaDatas);
 	m_MemCachedPayloads.swap(MemCachedPayloads);
 	Reset(m_FreeMemCachedPayloads);
-	if (m_Configuration.EnableReferenceCaching)
-	{
-		m_FirstReferenceIndex.swap(FirstReferenceIndex);
-		CompactReferences(IndexLock);
-	}
 }
 
 #if ZEN_WITH_TESTS