remove reference caching (#658)

* remove reference caching

7 files changed, 197 insertions, 698 deletions

diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index 95c85d6c8..4cf7c9a01 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -740,7 +740,6 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque
 					ResponseWriter.AddInteger("MemCacheTargetFootprintBytes"sv, Info->Config.DiskLayerConfig.MemCacheTargetFootprintBytes);
 					ResponseWriter.AddInteger("MemCacheTrimIntervalSeconds"sv, Info->Config.DiskLayerConfig.MemCacheTrimIntervalSeconds);
 					ResponseWriter.AddInteger("MemCacheMaxAgeSeconds"sv, Info->Config.DiskLayerConfig.MemCacheMaxAgeSeconds);
-					ResponseWriter.AddBool("EnableReferenceCaching"sv, Info->Config.DiskLayerConfig.BucketConfig.EnableReferenceCaching);
 				}
 				ResponseWriter.EndObject();
 
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 012925b51..00581d758 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -444,9 +444,6 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled);
 	LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log");
 	LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log");
-	LuaOptions.AddOption("cache.referencecache"sv,
-						 ServerOptions.StructuredCacheConfig.EnableReferenceCaching,
-						 "cache-reference-cache-enabled");
 
 	LuaOptions.AddOption("cache.memlayer.sizethreshold"sv,
 						 ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold,
@@ -856,13 +853,6 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"),
 					   "");
 
-	options.add_option("cache",
-					   "",
-					   "cache-reference-cache-enabled",
-					   "Whether caching of references is enabled",
-					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.EnableReferenceCaching)->default_value("false"),
-					   "");
-
 	options.add_option(
 		"cache",
 		"",
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index b5314b600..1e44d54c0 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -112,7 +112,6 @@ struct ZenStructuredCacheConfig
 	bool	 Enabled				 = true;
 	bool	 WriteLogEnabled		 = false;
 	bool	 AccessLogEnabled		 = false;
-	bool	 EnableReferenceCaching	 = false;
 	uint64_t MemCacheSizeThreshold	 = 1 * 1024;
 	uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024;
 	uint64_t MemTrimIntervalSeconds	 = 60;
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 37b3f0531..3f3762dbf 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -486,11 +486,10 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 	Config.AllowAutomaticCreationOfNamespaces = true;
 	Config.Logging							  = {.EnableWriteLog  = ServerOptions.StructuredCacheConfig.WriteLogEnabled,
 												 .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled};
-	Config.NamespaceConfig.DiskLayerConfig.BucketConfig.EnableReferenceCaching = ServerOptions.StructuredCacheConfig.EnableReferenceCaching;
-	Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold  = ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold,
-	Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes;
-	Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds	= ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds;
-	Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds		= ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds;
+	Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold,
+	Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes		  = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes;
+	Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds		  = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds;
+	Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds			  = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds;
 
 	if (ServerOptions.IsDedicated)
 	{
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 615f8640f..7274626ee 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -957,11 +957,6 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const
 
 	m_AccessTimes.resize(EntryCount, AccessTime(GcClock::TickCount()));
 
-	if (m_Configuration.EnableReferenceCaching)
-	{
-		m_FirstReferenceIndex.resize(EntryCount);
-	}
-
 	OutVersion = CacheBucketIndexHeader::Version2;
 	return Header.LogPosition;
 }
@@ -1023,11 +1018,6 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std::
 
 	m_AccessTimes.resize(m_Payloads.size(), AccessTime(GcClock::TickCount()));
 
-	if (m_Configuration.EnableReferenceCaching)
-	{
-		m_FirstReferenceIndex.resize(m_Payloads.size());
-	}
-
 	if (InvalidEntryCount)
 	{
 		ZEN_WARN("found {} invalid entries in '{}'", InvalidEntryCount, m_BucketDir);
@@ -1050,10 +1040,6 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco
 	m_FreeMetaDatas.clear();
 	m_MemCachedPayloads.clear();
 	m_FreeMemCachedPayloads.clear();
-	m_FirstReferenceIndex.clear();
-	m_ReferenceHashes.clear();
-	m_NextReferenceHashesIndexes.clear();
-	m_ReferenceCount = 0;
 
 	std::filesystem::path LogPath	= GetLogPath(m_BucketDir, m_BucketName);
 	std::filesystem::path IndexPath = GetIndexPath(m_BucketDir, m_BucketName);
@@ -1419,10 +1405,6 @@ ZenCacheDiskLayer::CacheBucket::Drop()
 	m_FreeMetaDatas.clear();
 	m_MemCachedPayloads.clear();
 	m_FreeMemCachedPayloads.clear();
-	m_FirstReferenceIndex.clear();
-	m_ReferenceHashes.clear();
-	m_NextReferenceHashesIndexes.clear();
-	m_ReferenceCount = 0;
 	m_StandaloneSize.store(0);
 	m_OuterCacheMemoryUsage.fetch_sub(m_MemCachedSize.load());
 	m_MemCachedSize.store(0);
@@ -1738,13 +1720,9 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 				for (const IoHash& BadKey : BadKeys)
 				{
 					// Log a tombstone and delete the in-memory index for the bad entry
-					const auto	   It	   = m_Index.find(BadKey);
-					BucketPayload& Payload = m_Payloads[It->second];
-					if (m_Configuration.EnableReferenceCaching)
-					{
-						RemoveReferences(IndexLock, m_FirstReferenceIndex[It->second]);
-					}
-					DiskLocation Location = Payload.Location;
+					const auto	   It		= m_Index.find(BadKey);
+					BucketPayload& Payload	= m_Payloads[It->second];
+					DiskLocation   Location = Payload.Location;
 					if (Location.IsFlagSet(DiskLocation::kStandaloneFile))
 					{
 						m_StandaloneSize.fetch_sub(Location.Size(), std::memory_order::relaxed);
@@ -1782,12 +1760,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 				std::vector<AccessTime>		AccessTimes;
 				std::vector<BucketMetaData> MetaDatas;
 				std::vector<MemCacheData>	MemCachedPayloads;
-				std::vector<ReferenceIndex> FirstReferenceIndex;
 				IndexMap					Index;
 
 				{
 					RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-					CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+					CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index);
 				}
 			}
 		}
@@ -1835,10 +1812,9 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 
 	const GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count();
 
-	IndexMap					Index;
-	std::vector<AccessTime>		AccessTimes;
-	std::vector<BucketPayload>	Payloads;
-	std::vector<ReferenceIndex> FirstReferenceIndex;
+	IndexMap				   Index;
+	std::vector<AccessTime>	   AccessTimes;
+	std::vector<BucketPayload> Payloads;
 	{
 		RwLock::SharedLockScope __(m_IndexLock);
 #if CALCULATE_BLOCKING_TIME
@@ -1853,10 +1829,9 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 		{
 			return;
 		}
-		Index				= m_Index;
-		AccessTimes			= m_AccessTimes;
-		Payloads			= m_Payloads;
-		FirstReferenceIndex = m_FirstReferenceIndex;
+		Index		= m_Index;
+		AccessTimes = m_AccessTimes;
+		Payloads	= m_Payloads;
 	}
 
 	std::vector<IoHash> ExpiredKeys;
@@ -1893,40 +1868,6 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 		{
 			continue;
 		}
-		if (m_Configuration.EnableReferenceCaching)
-		{
-			if (FirstReferenceIndex.empty() || FirstReferenceIndex[PayloadIndex] == ReferenceIndex::Unknown())
-			{
-				StructuredItemsWithUnknownAttachments.push_back(Entry);
-				continue;
-			}
-
-			bool ReferencesAreKnown = false;
-			{
-				RwLock::SharedLockScope IndexLock(m_IndexLock);
-#if CALCULATE_BLOCKING_TIME
-				Stopwatch  Timer;
-				const auto ___ = MakeGuard([&] {
-					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
-					WriteBlockTimeUs += ElapsedUs;
-					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
-				});
-#endif	// CALCULATE_BLOCKING_TIME
-				if (auto It = m_Index.find(Key); It != m_Index.end())
-				{
-					ReferencesAreKnown = GetReferences(IndexLock, m_FirstReferenceIndex[It->second], Cids);
-				}
-			}
-			if (ReferencesAreKnown)
-			{
-				if (Cids.size() >= 1024)
-				{
-					GcCtx.AddRetainedCids(Cids);
-					Cids.clear();
-				}
-				continue;
-			}
-		}
 		StructuredItemsWithUnknownAttachments.push_back(Entry);
 	}
 
@@ -1980,34 +1921,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 			ZEN_ASSERT(Buffer);
 			ZEN_ASSERT(Buffer.GetContentType() == ZenContentType::kCbObject);
 			CbObjectView Obj(Buffer.GetData());
-			size_t		 CurrentCidCount = Cids.size();
 			Obj.IterateAttachments([&Cids](CbFieldView Field) { Cids.push_back(Field.AsAttachment()); });
-			if (m_Configuration.EnableReferenceCaching)
-			{
-				RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-#if CALCULATE_BLOCKING_TIME
-				Stopwatch  Timer;
-				const auto ___ = MakeGuard([&] {
-					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
-					ReadBlockTimeUs += ElapsedUs;
-					ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
-				});
-#endif	// CALCULATE_BLOCKING_TIME
-				if (auto It = m_Index.find(Key); It != m_Index.end())
-				{
-					if (m_FirstReferenceIndex[It->second] == ReferenceIndex::Unknown())
-					{
-						SetReferences(IndexLock,
-									  m_FirstReferenceIndex[It->second],
-									  std::span<IoHash>(Cids.data() + CurrentCidCount, Cids.size() - CurrentCidCount));
-					}
-					else
-					{
-						Cids.resize(CurrentCidCount);
-						(void)GetReferences(IndexLock, m_FirstReferenceIndex[It->second], Cids);
-					}
-				}
-			}
 			if (Cids.size() >= 1024)
 			{
 				GcCtx.AddRetainedCids(Cids);
@@ -2083,7 +1997,6 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			std::vector<AccessTime>		AccessTimes;
 			std::vector<BucketMetaData> MetaDatas;
 			std::vector<MemCacheData>	MemCachedPayloads;
-			std::vector<ReferenceIndex> FirstReferenceIndex;
 			IndexMap					Index;
 			{
 				RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
@@ -2093,7 +2006,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 					WriteBlockTimeUs += ElapsedUs;
 					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
 				});
-				CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+				CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index);
 			}
 			GcCtx.AddDeletedCids(std::vector<IoHash>(DeletedChunks.begin(), DeletedChunks.end()));
 		}
@@ -2549,9 +2462,13 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 
 	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 	ValueLock.ReleaseNow();
-	if (m_UpdatedKeys)
+	if (m_TrackedCacheKeys)
 	{
-		m_UpdatedKeys->insert(HashKey);
+		m_TrackedCacheKeys->insert(HashKey);
+	}
+	if (m_TrackedReferences)
+	{
+		m_TrackedReferences->insert(References.begin(), References.end());
 	}
 
 	PayloadIndex EntryIndex = {};
@@ -2561,24 +2478,15 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 		EntryIndex = PayloadIndex(m_Payloads.size());
 		m_Payloads.emplace_back(BucketPayload{.Location = Loc});
 		m_AccessTimes.emplace_back(GcClock::TickCount());
-		if (m_Configuration.EnableReferenceCaching)
-		{
-			m_FirstReferenceIndex.emplace_back(ReferenceIndex{});
-			SetReferences(IndexLock, m_FirstReferenceIndex.back(), References);
-		}
 		m_Index.insert_or_assign(HashKey, EntryIndex);
 	}
 	else
 	{
 		EntryIndex = It.value();
 		ZEN_ASSERT_SLOW(EntryIndex < PayloadIndex(m_AccessTimes.size()));
-		BucketPayload& Payload = m_Payloads[EntryIndex];
-		uint64_t	   OldSize = Payload.Location.Size();
-		Payload				   = BucketPayload{.Location = Loc};
-		if (m_Configuration.EnableReferenceCaching)
-		{
-			SetReferences(IndexLock, m_FirstReferenceIndex[EntryIndex], References);
-		}
+		BucketPayload& Payload	  = m_Payloads[EntryIndex];
+		uint64_t	   OldSize	  = Payload.Location.Size();
+		Payload					  = BucketPayload{.Location = Loc};
 		m_AccessTimes[EntryIndex] = GcClock::TickCount();
 		RemoveMemCachedData(IndexLock, Payload);
 		m_StandaloneSize.fetch_sub(OldSize, std::memory_order::relaxed);
@@ -2706,9 +2614,13 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 								m_SlogFile.Append({.Key = HashKey, .Location = Location});
 
 								RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-								if (m_UpdatedKeys)
+								if (m_TrackedCacheKeys)
+								{
+									m_TrackedCacheKeys->insert(HashKey);
+								}
+								if (m_TrackedReferences)
 								{
-									m_UpdatedKeys->insert(HashKey);
+									m_TrackedReferences->insert(References.begin(), References.end());
 								}
 								if (auto It = m_Index.find(HashKey); It != m_Index.end())
 								{
@@ -2721,22 +2633,12 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 
 									Payload					  = (BucketPayload{.Location = Location});
 									m_AccessTimes[EntryIndex] = GcClock::TickCount();
-
-									if (m_Configuration.EnableReferenceCaching)
-									{
-										SetReferences(IndexLock, m_FirstReferenceIndex[EntryIndex], References);
-									}
 								}
 								else
 								{
 									PayloadIndex EntryIndex = PayloadIndex(m_Payloads.size());
 									m_Payloads.emplace_back(BucketPayload{.Location = Location});
 									m_AccessTimes.emplace_back(GcClock::TickCount());
-									if (m_Configuration.EnableReferenceCaching)
-									{
-										m_FirstReferenceIndex.emplace_back(ReferenceIndex{});
-										SetReferences(IndexLock, m_FirstReferenceIndex.back(), References);
-									}
 									m_Index.insert_or_assign(HashKey, EntryIndex);
 								}
 							});
@@ -2750,8 +2652,12 @@ ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&)
 
 class DiskBucketStoreCompactor : public GcStoreCompactor
 {
+	using PayloadIndex	= ZenCacheDiskLayer::CacheBucket::PayloadIndex;
+	using BucketPayload = ZenCacheDiskLayer::CacheBucket::BucketPayload;
+	using CacheBucket	= ZenCacheDiskLayer::CacheBucket;
+
 public:
-	DiskBucketStoreCompactor(ZenCacheDiskLayer::CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
+	DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
 	: m_Bucket(Bucket)
 	, m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys))
 	{
@@ -2854,8 +2760,8 @@ public:
 
 		if (Ctx.Settings.CollectSmallObjects)
 		{
-			m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_UpdatedKeys = std::make_unique<HashSet>(); });
-			auto __ = MakeGuard([&]() { m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_UpdatedKeys.reset(); }); });
+			m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_TrackedCacheKeys = std::make_unique<HashSet>(); });
+			auto __ = MakeGuard([&]() { m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_TrackedCacheKeys.reset(); }); });
 
 			size_t					  InlineEntryCount = 0;
 			BlockStore::BlockUsageMap BlockUsage;
@@ -2863,9 +2769,9 @@ public:
 				RwLock::SharedLockScope ___(m_Bucket.m_IndexLock);
 				for (const auto& Entry : m_Bucket.m_Index)
 				{
-					ZenCacheDiskLayer::CacheBucket::PayloadIndex		 Index	 = Entry.second;
-					const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = m_Bucket.m_Payloads[Index];
-					const DiskLocation&									 Loc	 = Payload.Location;
+					PayloadIndex		 Index	 = Entry.second;
+					const BucketPayload& Payload = m_Bucket.m_Payloads[Index];
+					const DiskLocation&	 Loc	 = Payload.Location;
 
 					if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 					{
@@ -2901,9 +2807,9 @@ public:
 						RwLock::SharedLockScope ___(m_Bucket.m_IndexLock);
 						for (const auto& Entry : m_Bucket.m_Index)
 						{
-							ZenCacheDiskLayer::CacheBucket::PayloadIndex		 Index	 = Entry.second;
-							const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = m_Bucket.m_Payloads[Index];
-							const DiskLocation&									 Loc	 = Payload.Location;
+							PayloadIndex		 Index	 = Entry.second;
+							const BucketPayload& Payload = m_Bucket.m_Payloads[Index];
+							const DiskLocation&	 Loc	 = Payload.Location;
 
 							if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 							{
@@ -2938,18 +2844,19 @@ public:
 									size_t		  ChunkIndex = Moved.first;
 									const IoHash& Key		 = BlockCompactStateKeys[ChunkIndex];
 
-									if (m_Bucket.m_UpdatedKeys->contains(Key))
+									ZEN_ASSERT(m_Bucket.m_TrackedCacheKeys);
+									if (m_Bucket.m_TrackedCacheKeys->contains(Key))
 									{
 										continue;
 									}
 
 									if (auto It = m_Bucket.m_Index.find(Key); It != m_Bucket.m_Index.end())
 									{
-										ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload	   = m_Bucket.m_Payloads[It->second];
-										const BlockStoreLocation&					   NewLocation = Moved.second;
-										Payload.Location										   = DiskLocation(NewLocation,
-																		  m_Bucket.m_Configuration.PayloadAlignment,
-																		  Payload.Location.GetFlags());
+										BucketPayload&			  Payload	  = m_Bucket.m_Payloads[It->second];
+										const BlockStoreLocation& NewLocation = Moved.second;
+										Payload.Location					  = DiskLocation(NewLocation,
+																		 m_Bucket.m_Configuration.PayloadAlignment,
+																		 Payload.Location.GetFlags());
 										MovedEntries.push_back({.Key = Key, .Location = Payload.Location});
 									}
 								}
@@ -2991,17 +2898,31 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
-		if (!Ctx.Settings.Verbose)
+		if (Ctx.Settings.Verbose)
+		{
+			ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}",
+					 m_BucketDir,
+					 Stats.CheckedCount,
+					 Stats.FoundCount,
+					 Stats.DeletedCount,
+					 NiceBytes(Stats.FreedMemory),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		}
+		bool Expected = false;
+		if (m_IsFlushing || !m_IsFlushing.compare_exchange_strong(Expected, true))
 		{
 			return;
 		}
-		ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}",
-				 m_BucketDir,
-				 Stats.CheckedCount,
-				 Stats.FoundCount,
-				 Stats.DeletedCount,
-				 NiceBytes(Stats.FreedMemory),
-				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		auto FlushingGuard = MakeGuard([&] { m_IsFlushing.store(false); });
+
+		try
+		{
+			SaveSnapshot([]() { return 0; });
+		}
+		catch (std::exception& Ex)
+		{
+			ZEN_WARN("Failed to write index and manifest after RemoveExpiredData in '{}'. Reason: '{}'", m_BucketDir, Ex.what());
+		}
 	});
 
 	const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
@@ -3080,11 +3001,10 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 		std::vector<AccessTime>		AccessTimes;
 		std::vector<BucketMetaData> MetaDatas;
 		std::vector<MemCacheData>	MemCachedPayloads;
-		std::vector<ReferenceIndex> FirstReferenceIndex;
 		IndexMap					Index;
 		{
 			RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-			CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+			CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index);
 		}
 	}
 
@@ -3098,10 +3018,9 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 
 class DiskBucketReferenceChecker : public GcReferenceChecker
 {
-	using PayloadIndex	 = ZenCacheDiskLayer::CacheBucket::PayloadIndex;
-	using BucketPayload	 = ZenCacheDiskLayer::CacheBucket::BucketPayload;
-	using CacheBucket	 = ZenCacheDiskLayer::CacheBucket;
-	using ReferenceIndex = ZenCacheDiskLayer::CacheBucket::ReferenceIndex;
+	using PayloadIndex	= ZenCacheDiskLayer::CacheBucket::PayloadIndex;
+	using BucketPayload = ZenCacheDiskLayer::CacheBucket::BucketPayload;
+	using CacheBucket	= ZenCacheDiskLayer::CacheBucket;
 
 public:
 	DiskBucketReferenceChecker(CacheBucket& Owner) : m_CacheBucket(Owner) {}
@@ -3111,12 +3030,7 @@ public:
 		try
 		{
 			m_IndexLock.reset();
-			if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
-			{
-				m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
-				// If reference caching is not enabled, we temporarily used the data structure for reference caching, lets reset it
-				m_CacheBucket.ClearReferenceCache();
-			}
+			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
 		}
 		catch (std::exception& Ex)
 		{
@@ -3136,28 +3050,18 @@ public:
 			}
 			ZEN_INFO("GCV2: cachebucket [PRECACHE] '{}': found {} references in {}",
 					 m_CacheBucket.m_BucketDir,
-					 m_CacheBucket.m_ReferenceCount,
+					 m_PrecachedReferences.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		std::vector<IoHash> UpdateKeys;
-		std::vector<size_t> ReferenceCounts;
-		std::vector<IoHash> References;
-
-		auto GetAttachments = [&References, &ReferenceCounts](const void* CbObjectData) {
-			size_t		 CurrentReferenceCount = References.size();
+		auto GetAttachments = [&](const void* CbObjectData) {
 			CbObjectView Obj(CbObjectData);
-			Obj.IterateAttachments([&References](CbFieldView Field) { References.emplace_back(Field.AsAttachment()); });
-			ReferenceCounts.push_back(References.size() - CurrentReferenceCount);
+			Obj.IterateAttachments([&](CbFieldView Field) { m_PrecachedReferences.emplace_back(Field.AsAttachment()); });
 		};
 
 		// Refresh cache
 		{
-			// If reference caching is enabled the references will be updated at modification for us so we don't need to track modifications
-			if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
-			{
-				m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys = std::make_unique<HashSet>(); });
-			}
+			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<HashSet>(); });
 
 			std::vector<IoHash> StandaloneKeys;
 			{
@@ -3170,7 +3074,6 @@ public:
 					uint32_t Size;
 				};
 				std::vector<std::vector<InlineEntry>> EntriesPerBlock;
-				size_t								  UpdateCount = 0;
 				{
 					RwLock::SharedLockScope IndexLock(m_CacheBucket.m_IndexLock);
 					for (const auto& Entry : m_CacheBucket.m_Index)
@@ -3178,7 +3081,7 @@ public:
 						if (Ctx.IsCancelledFlag.load())
 						{
 							IndexLock.ReleaseNow();
-							m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
+							m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
 							return;
 						}
 
@@ -3190,12 +3093,6 @@ public:
 						{
 							continue;
 						}
-						if (m_CacheBucket.m_Configuration.EnableReferenceCaching &&
-							m_CacheBucket.m_FirstReferenceIndex[EntryIndex] != ReferenceIndex::Unknown())
-						{
-							continue;
-						}
-						UpdateCount++;
 						const IoHash& Key = Entry.first;
 						if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 						{
@@ -3222,8 +3119,6 @@ public:
 					}
 				}
 
-				UpdateKeys.reserve(UpdateCount);
-
 				for (auto It : BlockIndexToEntriesPerBlockIndex)
 				{
 					uint32_t BlockIndex = It.first;
@@ -3242,11 +3137,7 @@ public:
 						BasicFileBuffer BlockBuffer(BlockFile->GetBasicFile(), 32768);
 						for (const InlineEntry& InlineEntry : InlineEntries)
 						{
-							if ((InlineEntry.Offset + InlineEntry.Size) > BlockFileSize)
-							{
-								ReferenceCounts.push_back(0);
-							}
-							else
+							if ((InlineEntry.Offset + InlineEntry.Size) <= BlockFileSize)
 							{
 								MemoryView ChunkView = BlockBuffer.MakeView(InlineEntry.Size, InlineEntry.Offset);
 								if (ChunkView.GetSize() == InlineEntry.Size)
@@ -3255,92 +3146,29 @@ public:
 								}
 								else
 								{
-									std::vector<uint8_t> Buffer(InlineEntry.Size);
-									BlockBuffer.Read(Buffer.data(), InlineEntry.Size, InlineEntry.Offset);
-									GetAttachments(Buffer.data());
+									IoBuffer Buffer = BlockFile->GetChunk(InlineEntry.Offset, InlineEntry.Size);
+									GetAttachments(Buffer.GetData());
 								}
 							}
-							const IoHash& Key = InlineKeys[InlineEntry.InlineKeyIndex];
-							UpdateKeys.push_back(Key);
 						}
 					}
 				}
 			}
+			for (const IoHash& Key : StandaloneKeys)
 			{
-				for (const IoHash& Key : StandaloneKeys)
+				if (Ctx.IsCancelledFlag.load())
 				{
-					if (Ctx.IsCancelledFlag.load())
-					{
-						m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
-						return;
-					}
-
-					IoBuffer Buffer = m_CacheBucket.GetStandaloneCacheValue(ZenContentType::kCbObject, Key);
-					if (!Buffer)
-					{
-						continue;
-					}
-
-					GetAttachments(Buffer.GetData());
-					UpdateKeys.push_back(Key);
+					m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
+					return;
 				}
-			}
-		}
-
-		{
-			size_t					   ReferenceOffset = 0;
-			RwLock::ExclusiveLockScope IndexLock(m_CacheBucket.m_IndexLock);
 
-			if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
-			{
-				ZEN_ASSERT(m_CacheBucket.m_FirstReferenceIndex.empty());
-				ZEN_ASSERT(m_CacheBucket.m_ReferenceHashes.empty());
-				ZEN_ASSERT(m_CacheBucket.m_NextReferenceHashesIndexes.empty());
-				ZEN_ASSERT(m_CacheBucket.m_ReferenceCount == 0);
-				ZEN_ASSERT(m_CacheBucket.m_UpdatedKeys);
-
-				// If reference caching is not enabled, we will resize and use the data structure in place for reference caching when
-				// we figure out what this bucket references. This will be reset once the DiskBucketReferenceChecker is deleted.
-				m_CacheBucket.m_FirstReferenceIndex.resize(m_CacheBucket.m_Payloads.size(), ReferenceIndex::Unknown());
-				m_CacheBucket.m_ReferenceHashes.reserve(References.size());
-				m_CacheBucket.m_NextReferenceHashesIndexes.reserve(References.size());
-			}
-			else
-			{
-				ZEN_ASSERT(!m_CacheBucket.m_UpdatedKeys);
-			}
-
-			for (size_t Index = 0; Index < UpdateKeys.size(); Index++)
-			{
-				const IoHash& Key			 = UpdateKeys[Index];
-				size_t		  ReferenceCount = ReferenceCounts[Index];
-				if (auto It = m_CacheBucket.m_Index.find(Key); It != m_CacheBucket.m_Index.end())
+				IoBuffer Buffer = m_CacheBucket.GetStandaloneCacheValue(ZenContentType::kCbObject, Key);
+				if (!Buffer)
 				{
-					PayloadIndex EntryIndex = It->second;
-					if (m_CacheBucket.m_Configuration.EnableReferenceCaching)
-					{
-						if (m_CacheBucket.m_FirstReferenceIndex[EntryIndex] != ReferenceIndex::Unknown())
-						{
-							// The reference data is valid and what we have is old/redundant
-							continue;
-						}
-					}
-					else if (m_CacheBucket.m_UpdatedKeys->contains(Key))
-					{
-						// Our pre-cache data is invalid
-						continue;
-					}
-
-					m_CacheBucket.SetReferences(IndexLock,
-												m_CacheBucket.m_FirstReferenceIndex[EntryIndex],
-												std::span<IoHash>{References.data() + ReferenceOffset, ReferenceCount});
+					continue;
 				}
-				ReferenceOffset += ReferenceCount;
-			}
 
-			if (m_CacheBucket.m_Configuration.EnableReferenceCaching && !UpdateKeys.empty())
-			{
-				m_CacheBucket.CompactReferences(IndexLock);
+				GetAttachments(Buffer.GetData());
 			}
 		}
 	}
@@ -3357,65 +3185,25 @@ public:
 			}
 			ZEN_INFO("GCV2: cachebucket [LOCKSTATE] '{}': found {} references in {}",
 					 m_CacheBucket.m_BucketDir,
-					 m_CacheBucket.m_ReferenceCount + m_UncachedReferences.size(),
+					 m_PrecachedReferences.size() + m_UncachedReferences.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		m_IndexLock = std::make_unique<RwLock::SharedLockScope>(m_CacheBucket.m_IndexLock);
 		if (Ctx.IsCancelledFlag.load())
 		{
-			m_UncachedReferences.clear();
+			m_UncachedReferences = {};
 			m_IndexLock.reset();
-			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
+			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
 			return;
 		}
 
-		if (m_CacheBucket.m_UpdatedKeys)
-		{
-			const HashSet& UpdatedKeys(*m_CacheBucket.m_UpdatedKeys);
-			for (const IoHash& Key : UpdatedKeys)
-			{
-				if (Ctx.IsCancelledFlag.load())
-				{
-					m_UncachedReferences.clear();
-					m_IndexLock.reset();
-					m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
-					return;
-				}
+		ZEN_ASSERT(m_CacheBucket.m_TrackedReferences);
 
-				auto It = m_CacheBucket.m_Index.find(Key);
-				if (It == m_CacheBucket.m_Index.end())
-				{
-					continue;
-				}
-
-				PayloadIndex		 EntryIndex = It->second;
-				const BucketPayload& Payload	= m_CacheBucket.m_Payloads[EntryIndex];
-				const DiskLocation&	 Loc		= Payload.Location;
-
-				if (!Loc.IsFlagSet(DiskLocation::kStructured))
-				{
-					continue;
-				}
-
-				IoBuffer Buffer;
-				if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
-				{
-					Buffer = m_CacheBucket.GetStandaloneCacheValue(Loc.GetContentType(), Key);
-				}
-				else
-				{
-					Buffer = m_CacheBucket.GetInlineCacheValue(Loc);
-				}
-
-				if (Buffer)
-				{
-					ZEN_ASSERT(Buffer.GetContentType() == ZenContentType::kCbObject);
-					CbObjectView Obj(Buffer.GetData());
-					Obj.IterateAttachments([this](CbFieldView Field) { m_UncachedReferences.insert(Field.AsAttachment()); });
-				}
-			}
-		}
+		HashSet& AddedReferences(*m_CacheBucket.m_TrackedReferences);
+		m_UncachedReferences.reserve(AddedReferences.size());
+		m_UncachedReferences.insert(m_UncachedReferences.end(), AddedReferences.begin(), AddedReferences.end());
+		AddedReferences = {};
 	}
 
 	virtual void RemoveUsedReferencesFromSet(GcCtx& Ctx, HashSet& IoCids) override
@@ -3437,7 +3225,7 @@ public:
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		for (const IoHash& ReferenceHash : m_CacheBucket.m_ReferenceHashes)
+		for (const IoHash& ReferenceHash : m_PrecachedReferences)
 		{
 			if (IoCids.erase(ReferenceHash) == 1)
 			{
@@ -3461,7 +3249,8 @@ public:
 	}
 	CacheBucket&							 m_CacheBucket;
 	std::unique_ptr<RwLock::SharedLockScope> m_IndexLock;
-	HashSet									 m_UncachedReferences;
+	std::vector<IoHash>						 m_PrecachedReferences;
+	std::vector<IoHash>						 m_UncachedReferences;
 };
 
 std::vector<GcReferenceChecker*>
@@ -3490,200 +3279,18 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::CompactReferences(RwLock::ExclusiveLockScope&)
-{
-	ZEN_TRACE_CPU("Z$::Bucket::CompactReferences");
-
-	std::vector<ReferenceIndex> FirstReferenceIndex;
-	std::vector<IoHash>			NewReferenceHashes;
-	std::vector<ReferenceIndex> NewNextReferenceHashesIndexes;
-
-	FirstReferenceIndex.reserve(m_ReferenceCount);
-	NewReferenceHashes.reserve(m_ReferenceCount);
-	NewNextReferenceHashesIndexes.reserve(m_ReferenceCount);
-
-	for (const auto& It : m_Index)
-	{
-		ReferenceIndex SourceIndex = m_FirstReferenceIndex[It.second];
-		if (SourceIndex == ReferenceIndex::Unknown())
-		{
-			FirstReferenceIndex.push_back(ReferenceIndex{});
-			continue;
-		}
-		if (SourceIndex == ReferenceIndex::None())
-		{
-			FirstReferenceIndex.push_back(ReferenceIndex::None());
-			continue;
-		}
-		FirstReferenceIndex.push_back(ReferenceIndex{NewNextReferenceHashesIndexes.size()});
-		NewReferenceHashes.push_back(m_ReferenceHashes[SourceIndex]);
-		NewNextReferenceHashesIndexes.push_back(ReferenceIndex::None());
-
-		SourceIndex = m_NextReferenceHashesIndexes[SourceIndex];
-		while (SourceIndex != ReferenceIndex::None())
-		{
-			NewNextReferenceHashesIndexes.back() = ReferenceIndex{NewReferenceHashes.size()};
-			NewReferenceHashes.push_back(m_ReferenceHashes[SourceIndex]);
-			NewNextReferenceHashesIndexes.push_back(ReferenceIndex::None());
-			SourceIndex = m_NextReferenceHashesIndexes[SourceIndex];
-		}
-	}
-	m_FirstReferenceIndex.swap(FirstReferenceIndex);
-	m_ReferenceHashes.swap(NewReferenceHashes);
-	m_ReferenceHashes.shrink_to_fit();
-	m_NextReferenceHashesIndexes.swap(NewNextReferenceHashesIndexes);
-	m_NextReferenceHashesIndexes.shrink_to_fit();
-	m_ReferenceCount = m_ReferenceHashes.size();
-}
-
-ZenCacheDiskLayer::CacheBucket::ReferenceIndex
-ZenCacheDiskLayer::CacheBucket::AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key)
-{
-	ReferenceIndex NewIndex = ReferenceIndex{m_ReferenceHashes.size()};
-	m_ReferenceHashes.push_back(Key);
-	m_NextReferenceHashesIndexes.emplace_back(ReferenceIndex::None());
-	m_ReferenceCount++;
-	return NewIndex;
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::SetReferences(RwLock::ExclusiveLockScope& Lock,
-											  ReferenceIndex&			  FirstReferenceIndex,
-											  std::span<IoHash>			  References)
-{
-	auto ReferenceIt = References.begin();
-
-	if (FirstReferenceIndex == ReferenceIndex::Unknown())
-	{
-		FirstReferenceIndex = ReferenceIndex::None();
-	}
-
-	ReferenceIndex CurrentIndex = FirstReferenceIndex;
-	if (CurrentIndex != ReferenceIndex::None())
-	{
-		if (ReferenceIt != References.end())
-		{
-			ZEN_ASSERT_SLOW(*ReferenceIt != IoHash::Zero);
-			if (CurrentIndex == ReferenceIndex::None())
-			{
-				CurrentIndex		= AllocateReferenceEntry(Lock, *ReferenceIt);
-				FirstReferenceIndex = CurrentIndex;
-			}
-			else
-			{
-				m_ReferenceHashes[CurrentIndex] = *ReferenceIt;
-			}
-			ReferenceIt++;
-		}
-	}
-	else
-	{
-		if (ReferenceIt != References.end())
-		{
-			ZEN_ASSERT_SLOW(*ReferenceIt != IoHash::Zero);
-			CurrentIndex = AllocateReferenceEntry(Lock, *ReferenceIt);
-			ReferenceIt++;
-		}
-		FirstReferenceIndex = CurrentIndex;
-	}
-
-	while (ReferenceIt != References.end())
-	{
-		ZEN_ASSERT(CurrentIndex != ReferenceIndex::None());
-		ZEN_ASSERT_SLOW(*ReferenceIt != IoHash::Zero);
-		ReferenceIndex NextReferenceIndex = m_NextReferenceHashesIndexes[CurrentIndex];
-		if (NextReferenceIndex == ReferenceIndex::None())
-		{
-			NextReferenceIndex						   = AllocateReferenceEntry(Lock, *ReferenceIt);
-			m_NextReferenceHashesIndexes[CurrentIndex] = NextReferenceIndex;
-		}
-		else
-		{
-			m_ReferenceHashes[NextReferenceIndex] = *ReferenceIt;
-		}
-		CurrentIndex = NextReferenceIndex;
-		ReferenceIt++;
-	}
-
-	while (CurrentIndex != ReferenceIndex::None())
-	{
-		ReferenceIndex NextIndex = m_NextReferenceHashesIndexes[CurrentIndex];
-		if (NextIndex != ReferenceIndex::None())
-		{
-			m_ReferenceHashes[CurrentIndex] = IoHash::Zero;
-			ZEN_ASSERT(m_ReferenceCount > 0);
-			m_ReferenceCount--;
-			m_NextReferenceHashesIndexes[CurrentIndex] = ReferenceIndex::None();
-		}
-		CurrentIndex = NextIndex;
-	}
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::RemoveReferences(RwLock::ExclusiveLockScope&, ReferenceIndex& FirstReferenceIndex)
-{
-	if (FirstReferenceIndex == ReferenceIndex::Unknown())
-	{
-		return;
-	}
-	ReferenceIndex CurrentIndex = FirstReferenceIndex;
-	while (CurrentIndex == ReferenceIndex::None())
-	{
-		m_ReferenceHashes[CurrentIndex] = IoHash::Zero;
-		ZEN_ASSERT(m_ReferenceCount > 0);
-		m_ReferenceCount--;
-		CurrentIndex = m_NextReferenceHashesIndexes[CurrentIndex];
-	}
-	FirstReferenceIndex = {};
-}
-
-bool
-ZenCacheDiskLayer::CacheBucket::LockedGetReferences(ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
-{
-	if (FirstReferenceIndex == ReferenceIndex::Unknown())
-	{
-		return false;
-	}
-
-	ReferenceIndex CurrentIndex = FirstReferenceIndex;
-	while (CurrentIndex != ReferenceIndex::None())
-	{
-		ZEN_ASSERT_SLOW(m_ReferenceHashes[CurrentIndex] != IoHash::Zero);
-		OutReferences.push_back(m_ReferenceHashes[CurrentIndex]);
-		CurrentIndex = m_NextReferenceHashesIndexes[CurrentIndex];
-	}
-	return true;
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::ClearReferenceCache()
-{
-	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-	Reset(m_FirstReferenceIndex);
-	Reset(m_ReferenceHashes);
-	Reset(m_NextReferenceHashesIndexes);
-	m_ReferenceCount = 0;
-}
-
-void
 ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&,
 											 std::vector<BucketPayload>&  Payloads,
 											 std::vector<AccessTime>&	  AccessTimes,
 											 std::vector<BucketMetaData>& MetaDatas,
 											 std::vector<MemCacheData>&	  MemCachedPayloads,
-											 std::vector<ReferenceIndex>& FirstReferenceIndex,
-											 IndexMap&					  Index,
-											 RwLock::ExclusiveLockScope&  IndexLock)
+											 IndexMap&					  Index)
 {
 	ZEN_TRACE_CPU("Z$::Bucket::CompactState");
 
 	size_t EntryCount = m_Index.size();
 	Payloads.reserve(EntryCount);
 	AccessTimes.reserve(EntryCount);
-	if (m_Configuration.EnableReferenceCaching)
-	{
-		FirstReferenceIndex.reserve(EntryCount);
-	}
 	Index.reserve(EntryCount);
 	Index.min_load_factor(IndexMinLoadFactor);
 	Index.max_load_factor(IndexMaxLoadFactor);
@@ -3704,10 +3311,6 @@ ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&,
 				MemCacheData{.Payload = std::move(m_MemCachedPayloads[Payload.MemCached].Payload), .OwnerIndex = EntryIndex});
 			Payload.MemCached = MemCachedIndex(gsl::narrow<uint32_t>(MemCachedPayloads.size() - 1));
 		}
-		if (m_Configuration.EnableReferenceCaching)
-		{
-			FirstReferenceIndex.push_back(m_FirstReferenceIndex[It.second]);
-		}
 		Index.insert({It.first, EntryIndex});
 	}
 	m_Index.swap(Index);
@@ -3717,11 +3320,6 @@ ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&,
 	Reset(m_FreeMetaDatas);
 	m_MemCachedPayloads.swap(MemCachedPayloads);
 	Reset(m_FreeMemCachedPayloads);
-	if (m_Configuration.EnableReferenceCaching)
-	{
-		m_FirstReferenceIndex.swap(FirstReferenceIndex);
-		CompactReferences(IndexLock);
-	}
 }
 
 #if ZEN_WITH_TESTS
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index 49183600d..32a2a2aa3 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -882,17 +882,14 @@ namespace testutils {
 
 }  // namespace testutils
 
-TEST_CASE_TEMPLATE("z$.store", ReferenceCaching, testutils::FalseType, testutils::TrueType)
+TEST_CASE("z$.store")
 {
 	ScopedTemporaryDirectory TempDir;
 
 	GcManager Gc;
 
 	auto			  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace Zcs(Gc,
-						  *JobQueue,
-						  TempDir.Path() / "cache",
-						  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 
 	const int kIterationCount = 100;
 
@@ -926,7 +923,7 @@ TEST_CASE_TEMPLATE("z$.store", ReferenceCaching, testutils::FalseType, testutils
 	}
 }
 
-TEST_CASE_TEMPLATE("z$.size", ReferenceCaching, testutils::FalseType, testutils::TrueType)
+TEST_CASE("z$.size")
 {
 	auto JobQueue = MakeJobQueue(1, "testqueue");
 
@@ -948,10 +945,7 @@ TEST_CASE_TEMPLATE("z$.size", ReferenceCaching, testutils::FalseType, testutils:
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc,
-								  *JobQueue,
-								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 
 			CbObject CacheValue = CreateCacheValue(Zcs.GetConfig().DiskLayerConfig.BucketConfig.MemCacheSizeThreshold - 256);
 
@@ -985,10 +979,7 @@ TEST_CASE_TEMPLATE("z$.size", ReferenceCaching, testutils::FalseType, testutils:
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc,
-								  *JobQueue,
-								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -1012,10 +1003,7 @@ TEST_CASE_TEMPLATE("z$.size", ReferenceCaching, testutils::FalseType, testutils:
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc,
-								  *JobQueue,
-								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 
 			CbObject CacheValue = CreateCacheValue(Zcs.GetConfig().DiskLayerConfig.BucketConfig.MemCacheSizeThreshold + 64);
 
@@ -1035,10 +1023,7 @@ TEST_CASE_TEMPLATE("z$.size", ReferenceCaching, testutils::FalseType, testutils:
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc,
-								  *JobQueue,
-								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -1053,7 +1038,7 @@ TEST_CASE_TEMPLATE("z$.size", ReferenceCaching, testutils::FalseType, testutils:
 	}
 }
 
-TEST_CASE_TEMPLATE("z$.gc", ReferenceCaching, testutils::FalseType, testutils::TrueType)
+TEST_CASE("z$.gc")
 {
 	using namespace testutils;
 
@@ -1077,10 +1062,7 @@ TEST_CASE_TEMPLATE("z$.gc", ReferenceCaching, testutils::FalseType, testutils::T
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc,
-								  *JobQueue,
-								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 			const auto		  Bucket = "teardrinker"sv;
 
 			// Create a cache record
@@ -1117,10 +1099,7 @@ TEST_CASE_TEMPLATE("z$.gc", ReferenceCaching, testutils::FalseType, testutils::T
 		// Expect timestamps to be serialized
 		{
 			GcManager			Gc;
-			ZenCacheNamespace	Zcs(Gc,
-									*JobQueue,
-									TempDir.Path() / "cache",
-									{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+			ZenCacheNamespace	Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 			std::vector<IoHash> Keep;
 
 			// Collect garbage with 1 hour max cache duration
@@ -1141,10 +1120,7 @@ TEST_CASE_TEMPLATE("z$.gc", ReferenceCaching, testutils::FalseType, testutils::T
 	{
 		ScopedTemporaryDirectory TempDir;
 		GcManager				 Gc;
-		ZenCacheNamespace		 Zcs(Gc,
-								 *JobQueue,
-								 TempDir.Path() / "cache",
-								 {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace		 Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		const auto				 Bucket		 = "fortysixandtwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -1190,10 +1166,7 @@ TEST_CASE_TEMPLATE("z$.gc", ReferenceCaching, testutils::FalseType, testutils::T
 		ScopedTemporaryDirectory TempDir;
 		GcManager				 Gc;
 		{
-			ZenCacheNamespace Zcs(Gc,
-								  *JobQueue,
-								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 			const auto		  Bucket = "rightintwo"sv;
 
 			std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
@@ -1238,16 +1211,13 @@ TEST_CASE_TEMPLATE("z$.gc", ReferenceCaching, testutils::FalseType, testutils::T
 		}
 		{
 			// Unreferenced blocks will be pruned so size should now be zero
-			ZenCacheNamespace Zcs(Gc,
-								  *JobQueue,
-								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 			CHECK_EQ(0, Zcs.StorageSize().DiskSize);
 		}
 	}
 }
 
-TEST_CASE_TEMPLATE("z$.threadedinsert", ReferenceCaching, testutils::FalseType, testutils::TrueType)  // * doctest::skip(true))
+TEST_CASE_TEMPLATE("z$.threadedinsert", GCV2, testutils::FalseType, testutils::TrueType)  // * doctest::skip(true))
 {
 	//	for (uint32_t i = 0; i < 100; ++i)
 	{
@@ -1298,10 +1268,7 @@ TEST_CASE_TEMPLATE("z$.threadedinsert", ReferenceCaching, testutils::FalseType,
 		WorkerThreadPool  ThreadPool(4);
 		GcManager		  Gc;
 		auto			  JobQueue = MakeJobQueue(1, "testqueue");
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path(),
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path(), {});
 
 		{
 			std::atomic<size_t> WorkCompleted = 0;
@@ -1318,6 +1285,41 @@ TEST_CASE_TEMPLATE("z$.threadedinsert", ReferenceCaching, testutils::FalseType,
 			}
 		}
 
+		auto DoGC = [](GcManager&												Gc,
+					   ZenCacheNamespace&										Zcs,
+					   std::unordered_map<IoHash, std::string, IoHash::Hasher>& GcChunkHashes,
+					   const std::vector<IoHash>&								KeepHashes) {
+			if (GCV2::Enabled)
+			{
+				GcSettings Settings = {.CacheExpireTime					  = GcClock::Now() - std::chrono::hours(24),
+									   .ProjectStoreExpireTime			  = GcClock::Now() - std::chrono::hours(24),
+									   .CollectSmallObjects				  = true,
+									   .IsDeleteMode					  = true,
+									   .CompactBlockUsageThresholdPercent = 100};
+				Gc.CollectGarbage(Settings);
+				// Cheating as we don't get the list of deleted hashes back from this call
+				std::unordered_map<IoHash, std::string, IoHash::Hasher> RemainingChunkHashes;
+				for (const auto& It : GcChunkHashes)
+				{
+					ZenCacheValue Tmp;
+					if (Zcs.Get(It.second, It.first, Tmp))
+					{
+						RemainingChunkHashes.insert(It);
+					}
+				}
+				GcChunkHashes.swap(RemainingChunkHashes);
+			}
+			else
+			{
+				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
+				GcCtx.CollectSmallObjects(true);
+				GcCtx.AddRetainedCids(KeepHashes);
+				Zcs.CollectGarbage(GcCtx);
+				const HashKeySet& Deleted = GcCtx.DeletedCids();
+				Deleted.IterateHashes([&GcChunkHashes](const IoHash& ChunkHash) { GcChunkHashes.erase(ChunkHash); });
+			}
+		};
+
 		const uint64_t TotalSize = Zcs.StorageSize().DiskSize;
 		CHECK_LE(kChunkSize * Chunks.size(), TotalSize);
 
@@ -1422,12 +1424,7 @@ TEST_CASE_TEMPLATE("z$.threadedinsert", ReferenceCaching, testutils::FalseType,
 					C++;
 				}
 
-				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-				GcCtx.CollectSmallObjects(true);
-				GcCtx.AddRetainedCids(KeepHashes);
-				Zcs.CollectGarbage(GcCtx);
-				const HashKeySet& Deleted = GcCtx.DeletedCids();
-				Deleted.IterateHashes([&GcChunkHashes](const IoHash& ChunkHash) { GcChunkHashes.erase(ChunkHash); });
+				DoGC(Gc, Zcs, GcChunkHashes, KeepHashes);
 			}
 
 			while (WorkCompleted < NewChunks.size() + Chunks.size())
@@ -1470,28 +1467,25 @@ TEST_CASE_TEMPLATE("z$.threadedinsert", ReferenceCaching, testutils::FalseType,
 					C++;
 				}
 
-				GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-				GcCtx.CollectSmallObjects(true);
-				GcCtx.AddRetainedCids(KeepHashes);
-				Zcs.CollectGarbage(GcCtx);
-				const HashKeySet& Deleted = GcCtx.DeletedCids();
-				Deleted.IterateHashes([&GcChunkHashes](const IoHash& ChunkHash) { GcChunkHashes.erase(ChunkHash); });
+				DoGC(Gc, Zcs, GcChunkHashes, KeepHashes);
 			}
 		}
 		{
-			std::atomic<size_t> WorkCompleted = 0;
-			for (const auto& Chunk : GcChunkHashes)
-			{
-				ThreadPool.ScheduleWork([&Zcs, &WorkCompleted, Chunk]() {
-					ZenCacheValue CacheValue;
-					CHECK(Zcs.Get(Chunk.second, Chunk.first, CacheValue));
-					CHECK(Chunk.first == IoHash::HashBuffer(CacheValue.Value));
-					WorkCompleted.fetch_add(1);
-				});
-			}
-			while (WorkCompleted < GcChunkHashes.size())
 			{
-				Sleep(1);
+				std::atomic<size_t> WorkCompleted = 0;
+				for (const auto& Chunk : GcChunkHashes)
+				{
+					ThreadPool.ScheduleWork([&Zcs, &WorkCompleted, Chunk]() {
+						ZenCacheValue CacheValue;
+						CHECK(Zcs.Get(Chunk.second, Chunk.first, CacheValue));
+						CHECK(Chunk.first == IoHash::HashBuffer(CacheValue.Value));
+						WorkCompleted.fetch_add(1);
+					});
+				}
+				while (WorkCompleted < GcChunkHashes.size())
+				{
+					Sleep(1);
+				}
 			}
 		}
 	}
@@ -1730,7 +1724,7 @@ TEST_CASE("z$.drop.namespace")
 	}
 }
 
-TEST_CASE_TEMPLATE("z$.blocked.disklayer.put", ReferenceCaching, testutils::FalseType, testutils::TrueType)
+TEST_CASE("z$.blocked.disklayer.put")
 {
 	ScopedTemporaryDirectory TempDir;
 
@@ -1747,10 +1741,7 @@ TEST_CASE_TEMPLATE("z$.blocked.disklayer.put", ReferenceCaching, testutils::Fals
 
 	GcManager		  Gc;
 	auto			  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace Zcs(Gc,
-						  *JobQueue,
-						  TempDir.Path() / "cache",
-						  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 
 	CbObject CacheValue = CreateCacheValue(64 * 1024 + 64);
 
@@ -1786,7 +1777,7 @@ TEST_CASE_TEMPLATE("z$.blocked.disklayer.put", ReferenceCaching, testutils::Fals
 	CHECK(memcmp(NewView.GetData(), Buffer2.GetData(), NewView.GetSize()) == 0);
 }
 
-TEST_CASE_TEMPLATE("z$.scrub", ReferenceCaching, testutils::FalseType, testutils::TrueType)
+TEST_CASE("z$.scrub")
 {
 	ScopedTemporaryDirectory TempDir;
 
@@ -1845,10 +1836,7 @@ TEST_CASE_TEMPLATE("z$.scrub", ReferenceCaching, testutils::FalseType, testutils
 	GcManager			  Gc;
 	CidStore			  CidStore(Gc);
 	auto				  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace	  Zcs(Gc,
-						  *JobQueue,
-						  TempDir.Path() / "cache",
-						  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+	ZenCacheNamespace	  Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
 	CidStore.Initialize(CidConfig);
 
@@ -1883,7 +1871,7 @@ TEST_CASE_TEMPLATE("z$.scrub", ReferenceCaching, testutils::FalseType, testutils
 	CHECK(ScrubCtx.BadCids().GetSize() == 0);
 }
 
-TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, testutils::TrueType)
+TEST_CASE("z$.newgc.basics")
 {
 	using namespace testutils;
 
@@ -2000,10 +1988,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 
 		// Create some basic data
 		{
@@ -2034,10 +2019,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(1),
@@ -2068,10 +2050,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2102,10 +2081,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2136,10 +2112,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2171,10 +2144,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2206,10 +2176,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2247,10 +2214,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2283,10 +2247,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
@@ -2323,10 +2284,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
@@ -2362,10 +2320,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
@@ -2402,10 +2357,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		// Prime so we can check GC of memory layer
@@ -2455,10 +2407,7 @@ TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, te
 		GcManager Gc;
 		CidStore  CidStore(Gc);
 		CidStore.Initialize({.RootDirectory = TempDir.Path() / "cas"});
-		ZenCacheNamespace Zcs(Gc,
-							  *JobQueue,
-							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		auto Attachments =
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 6997a12e4..7aced67ad 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -105,11 +105,10 @@ class ZenCacheDiskLayer
 public:
 	struct BucketConfiguration
 	{
-		uint64_t MaxBlockSize			= 1ull << 30;
-		uint32_t PayloadAlignment		= 1u << 4;
-		uint64_t MemCacheSizeThreshold	= 1 * 1024;
-		uint64_t LargeObjectThreshold	= 128 * 1024;
-		bool	 EnableReferenceCaching = false;
+		uint64_t MaxBlockSize		   = 1ull << 30;
+		uint32_t PayloadAlignment	   = 1u << 4;
+		uint64_t MemCacheSizeThreshold = 1 * 1024;
+		uint64_t LargeObjectThreshold  = 128 * 1024;
 	};
 
 	struct Configuration
@@ -248,20 +247,6 @@ public:
 			inline auto operator<=>(const MemCachedIndex& Other) const = default;
 		};
 
-		struct ReferenceIndex
-		{
-			uint32_t Index = std::numeric_limits<uint32_t>::max();
-
-			static const ReferenceIndex Unknown() { return ReferenceIndex{std::numeric_limits<uint32_t>::max()}; }
-			static const ReferenceIndex None() { return ReferenceIndex{std::numeric_limits<uint32_t>::max() - 1}; }
-
-			ReferenceIndex() = default;
-			explicit ReferenceIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
-						operator size_t() const { return Index; };
-						operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
-			inline auto operator<=>(const ReferenceIndex& Other) const = default;
-		};
-
 		struct PayloadIndex
 		{
 			uint32_t Index = std::numeric_limits<uint32_t>::max();
@@ -330,11 +315,8 @@ public:
 		std::vector<MetaDataIndex>	m_FreeMetaDatas;
 		std::vector<MemCacheData>	m_MemCachedPayloads;
 		std::vector<MemCachedIndex> m_FreeMemCachedPayloads;
-		std::vector<ReferenceIndex> m_FirstReferenceIndex;
-		std::vector<IoHash>			m_ReferenceHashes;
-		std::vector<ReferenceIndex> m_NextReferenceHashesIndexes;
-		std::unique_ptr<HashSet>	m_UpdatedKeys;
-		size_t						m_ReferenceCount = 0;
+		std::unique_ptr<HashSet>	m_TrackedCacheKeys;
+		std::unique_ptr<HashSet>	m_TrackedReferences;
 		std::atomic_uint64_t		m_StandaloneSize{};
 		std::atomic_uint64_t		m_MemCachedSize{};
 
@@ -349,21 +331,6 @@ public:
 		IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const;
 		CacheValueDetails::ValueDetails GetValueDetails(RwLock::SharedLockScope&, const IoHash& Key, PayloadIndex Index) const;
 
-		void		CompactReferences(RwLock::ExclusiveLockScope&);
-		void		SetReferences(RwLock::ExclusiveLockScope&, ReferenceIndex& FirstReferenceIndex, std::span<IoHash> References);
-		void		RemoveReferences(RwLock::ExclusiveLockScope&, ReferenceIndex& FirstReferenceIndex);
-		inline bool GetReferences(RwLock::SharedLockScope&, ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
-		{
-			return LockedGetReferences(FirstReferenceIndex, OutReferences);
-		}
-		inline bool GetReferences(RwLock::ExclusiveLockScope&, ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
-		{
-			return LockedGetReferences(FirstReferenceIndex, OutReferences);
-		}
-		ReferenceIndex AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key);
-		bool		   LockedGetReferences(ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const;
-		void		   ClearReferenceCache();
-
 		void		   SetMetaData(RwLock::ExclusiveLockScope&,
 								   BucketPayload&										 Payload,
 								   const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData);
@@ -391,14 +358,12 @@ public:
 		}
 		void WriteIndexSnapshotLocked(const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
 
-		void CompactState(RwLock::ExclusiveLockScope&,
+		void CompactState(RwLock::ExclusiveLockScope&  IndexLock,
 						  std::vector<BucketPayload>&  Payloads,
 						  std::vector<AccessTime>&	   AccessTimes,
 						  std::vector<BucketMetaData>& MetaDatas,
 						  std::vector<MemCacheData>&   MemCachedPayloads,
-						  std::vector<ReferenceIndex>& FirstReferenceIndex,
-						  IndexMap&					   Index,
-						  RwLock::ExclusiveLockScope&  IndexLock);
+						  IndexMap&					   Index);
 
 		void AddMemCacheUsage(uint64_t ValueSize)
 		{