cache reference tracking (#455)

- Feature: Add caching of referenced CId content for structured cache records, this avoid disk thrashing when gathering references for GC
  - disabled by default, enable with `--cache-reference-cache-enabled`
- Improvement: Faster collection of referenced CId content in project store

9 files changed, 598 insertions, 188 deletions

diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 177d37aa9..ce5770763 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -164,7 +164,13 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
 
 //////////////////////////////////////////////////////////////////////////
 
-ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName) : m_BucketName(std::move(BucketName)), m_BucketId(Oid::Zero)
+const size_t ZenCacheDiskLayer::CacheBucket::UnknownReferencesIndex;
+const size_t ZenCacheDiskLayer::CacheBucket::NoReferencesIndex;
+
+ZenCacheDiskLayer::CacheBucket::CacheBucket(std::string BucketName, bool EnableReferenceCaching)
+: m_BucketName(std::move(BucketName))
+, m_BucketId(Oid::Zero)
+, m_EnableReferenceCaching(EnableReferenceCaching)
 {
 	if (m_BucketName.starts_with(std::string_view("legacy")) || m_BucketName.ends_with(std::string_view("shadermap")))
 	{
@@ -473,6 +479,10 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(const std::filesystem::path& Index
 												 sizeof(CacheBucketIndexHeader));
 
 							m_Payloads.reserve(Header.EntryCount);
+							if (m_EnableReferenceCaching)
+							{
+								m_FirstReferenceIndex.reserve(Header.EntryCount);
+							}
 							m_AccessTimes.reserve(Header.EntryCount);
 							m_Index.reserve(Header.EntryCount);
 
@@ -487,6 +497,10 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(const std::filesystem::path& Index
 								size_t EntryIndex = m_Payloads.size();
 								m_Payloads.emplace_back(BucketPayload{.Location = Entry.Location, .RawSize = 0, .RawHash = IoHash::Zero});
 								m_AccessTimes.emplace_back(GcClock::TickCount());
+								if (m_EnableReferenceCaching)
+								{
+									m_FirstReferenceIndex.emplace_back(UnknownReferencesIndex);
+								}
 								m_Index.insert_or_assign(Entry.Key, EntryIndex);
 								EntryCount++;
 							}
@@ -546,6 +560,10 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(const std::filesystem::path& LogPath, ui
 					size_t EntryIndex = m_Payloads.size();
 					m_Payloads.emplace_back(BucketPayload{.Location = Record.Location, .RawSize = 0u, .RawHash = IoHash::Zero});
 					m_AccessTimes.emplace_back(GcClock::TickCount());
+					if (m_EnableReferenceCaching)
+					{
+						m_FirstReferenceIndex.emplace_back(UnknownReferencesIndex);
+					}
 					m_Index.insert_or_assign(Record.Key, EntryIndex);
 				},
 				SkipEntryCount);
@@ -569,6 +587,10 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const bool IsNew)
 	m_Index.clear();
 	m_Payloads.clear();
 	m_AccessTimes.clear();
+	m_FirstReferenceIndex.clear();
+	m_ReferenceHashes.clear();
+	m_NextReferenceHashesIndexes.clear();
+	m_ReferenceCount = 0;
 
 	std::filesystem::path LogPath	= GetLogPath(m_BucketDir, m_BucketName);
 	std::filesystem::path IndexPath = GetIndexPath(m_BucketDir, m_BucketName);
@@ -711,7 +733,7 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc) con
 }
 
 IoBuffer
-ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const
+ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(ZenContentType ContentType, const IoHash& HashKey) const
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::GetStandaloneCacheValue");
 
@@ -722,7 +744,7 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc,
 
 	if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.ToPath()))
 	{
-		Data.SetContentType(Loc.GetContentType());
+		Data.SetContentType(ContentType);
 
 		return Data;
 	}
@@ -752,7 +774,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 	{
 		// We don't need to hold the index lock when we read a standalone file
 		_.ReleaseNow();
-		OutValue.Value = GetStandaloneCacheValue(Location, HashKey);
+		OutValue.Value = GetStandaloneCacheValue(Location.GetContentType(), HashKey);
 	}
 	else
 	{
@@ -798,15 +820,15 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
 {
 	metrics::RequestStats::Scope $(m_PutOps, Value.Value.Size());
 
 	if (Value.Value.Size() >= m_LargeObjectThreshold)
 	{
-		return PutStandaloneCacheValue(HashKey, Value);
+		return PutStandaloneCacheValue(HashKey, Value, References);
 	}
-	PutInlineCacheValue(HashKey, Value);
+	PutInlineCacheValue(HashKey, Value, References);
 
 	m_WriteCount++;
 }
@@ -832,6 +854,11 @@ ZenCacheDiskLayer::CacheBucket::Drop()
 	m_Index.clear();
 	m_Payloads.clear();
 	m_AccessTimes.clear();
+	m_FirstReferenceIndex.clear();
+	m_ReferenceHashes.clear();
+	m_NextReferenceHashesIndexes.clear();
+	m_ReferenceCount = 0;
+
 	return Deleted;
 }
 
@@ -1081,7 +1108,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 				else
 				{
 					// Structured cache value
-					IoBuffer Buffer = GetStandaloneCacheValue(Loc, HashKey);
+					IoBuffer Buffer = GetStandaloneCacheValue(Loc.GetContentType(), HashKey);
 					if (!Buffer)
 					{
 						ReportBadKey(HashKey);
@@ -1176,13 +1203,17 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 			LogEntries.reserve(BadKeys.size());
 
 			{
-				RwLock::ExclusiveLockScope __(m_IndexLock);
+				RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 				for (const IoHash& BadKey : BadKeys)
 				{
 					// Log a tombstone and delete the in-memory index for the bad entry
-					const auto			 It		  = m_Index.find(BadKey);
-					const BucketPayload& Payload  = m_Payloads[It->second];
-					DiskLocation		 Location = Payload.Location;
+					const auto	   It	   = m_Index.find(BadKey);
+					BucketPayload& Payload = m_Payloads[It->second];
+					if (m_EnableReferenceCaching)
+					{
+						RemoveReferences(IndexLock, m_FirstReferenceIndex[It->second]);
+					}
+					DiskLocation Location = Payload.Location;
 					Location.Flags |= DiskLocation::kTombStone;
 					LogEntries.push_back(DiskIndexEntry{.Key = BadKey, .Location = Location});
 					m_Index.erase(BadKey);
@@ -1211,6 +1242,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 			{
 				std::vector<BucketPayload> Payloads;
 				std::vector<AccessTime>	   AccessTimes;
+				std::vector<size_t>		   FirstReferenceIndex;
 				IndexMap				   Index;
 
 				{
@@ -1218,17 +1250,27 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 					size_t					   EntryCount = m_Index.size();
 					Payloads.reserve(EntryCount);
 					AccessTimes.reserve(EntryCount);
+					if (m_EnableReferenceCaching)
+					{
+						FirstReferenceIndex.reserve(EntryCount);
+					}
 					Index.reserve(EntryCount);
 					for (auto It : m_Index)
 					{
 						size_t EntryIndex = Payloads.size();
 						Payloads.push_back(m_Payloads[It.second]);
 						AccessTimes.push_back(m_AccessTimes[It.second]);
+						if (m_EnableReferenceCaching)
+						{
+							FirstReferenceIndex.push_back(m_FirstReferenceIndex[It.second]);
+						}
 						Index.insert({It.first, EntryIndex});
 					}
 					m_Index.swap(Index);
 					m_Payloads.swap(Payloads);
 					m_AccessTimes.swap(AccessTimes);
+					m_FirstReferenceIndex.swap(FirstReferenceIndex);
+					CompactReferences(__);
 				}
 			}
 		}
@@ -1248,13 +1290,18 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::GatherReferences");
 
+#define CALCULATE_BLOCKING_TIME 0
+
+#if CALCULATE_BLOCKING_TIME
 	uint64_t WriteBlockTimeUs		 = 0;
 	uint64_t WriteBlockLongestTimeUs = 0;
 	uint64_t ReadBlockTimeUs		 = 0;
 	uint64_t ReadBlockLongestTimeUs	 = 0;
+#endif	// CALCULATE_BLOCKING_TIME
 
 	Stopwatch  TotalTimer;
 	const auto _ = MakeGuard([&] {
+#if CALCULATE_BLOCKING_TIME
 		ZEN_DEBUG("gathered references from '{}' in {} write lock: {} ({}), read lock: {} ({})",
 				  m_BucketDir,
 				  NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()),
@@ -1262,6 +1309,9 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 				  NiceLatencyNs(WriteBlockLongestTimeUs),
 				  NiceLatencyNs(ReadBlockTimeUs),
 				  NiceLatencyNs(ReadBlockLongestTimeUs));
+#else
+		ZEN_DEBUG("gathered references from '{}' in {}", m_BucketDir, NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
+#endif	// CALCULATE_BLOCKING_TIME
 	});
 
 	const GcClock::TimePoint ExpireTime = GcCtx.CacheExpireTime();
@@ -1271,17 +1321,21 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 	IndexMap				   Index;
 	std::vector<AccessTime>	   AccessTimes;
 	std::vector<BucketPayload> Payloads;
+	std::vector<size_t>		   FirstReferenceIndex;
 	{
 		RwLock::SharedLockScope __(m_IndexLock);
-		Stopwatch				Timer;
-		const auto				___ = MakeGuard([&] {
-			 uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
-			 WriteBlockTimeUs += ElapsedUs;
-			 WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
-		 });
-		Index						= m_Index;
-		AccessTimes					= m_AccessTimes;
-		Payloads					= m_Payloads;
+#if CALCULATE_BLOCKING_TIME
+		Stopwatch  Timer;
+		const auto ___ = MakeGuard([&] {
+			uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
+			WriteBlockTimeUs += ElapsedUs;
+			WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
+		});
+#endif	// CALCULATE_BLOCKING_TIME
+		Index				= m_Index;
+		AccessTimes			= m_AccessTimes;
+		Payloads			= m_Payloads;
+		FirstReferenceIndex = m_FirstReferenceIndex;
 	}
 
 	std::vector<IoHash> ExpiredKeys;
@@ -1293,6 +1347,8 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 		Cids.reserve(1024);
 	}
 
+	std::vector<std::pair<IoHash, size_t>> StructuredItemsWithUnknownAttachments;
+
 	for (const auto& Entry : Index)
 	{
 		const IoHash& Key		 = Entry.first;
@@ -1308,35 +1364,80 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 			continue;
 		}
 
-		const DiskLocation& Loc = Payloads[Entry.second].Location;
+		BucketPayload&		Payload = Payloads[Entry.second];
+		const DiskLocation& Loc		= Payload.Location;
 
-		if (Loc.IsFlagSet(DiskLocation::kStructured))
+		if (!Loc.IsFlagSet(DiskLocation::kStructured))
 		{
-			if (Cids.size() > 1024)
+			continue;
+		}
+		if (m_EnableReferenceCaching)
+		{
+			if (FirstReferenceIndex.empty() || (FirstReferenceIndex[Entry.second] == UnknownReferencesIndex))
 			{
-				GcCtx.AddRetainedCids(Cids);
-				Cids.clear();
+				StructuredItemsWithUnknownAttachments.push_back(Entry);
+				continue;
 			}
 
+			bool ReferencesAreKnown = false;
+			{
+				RwLock::SharedLockScope IndexLock(m_IndexLock);
+#if CALCULATE_BLOCKING_TIME
+				Stopwatch  Timer;
+				const auto ___ = MakeGuard([&] {
+					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
+					WriteBlockTimeUs += ElapsedUs;
+					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
+				});
+#endif	// CALCULATE_BLOCKING_TIME
+				if (auto It = m_Index.find(Entry.first); It != m_Index.end())
+				{
+					ReferencesAreKnown = GetReferences(IndexLock, m_FirstReferenceIndex[It->second], Cids);
+				}
+			}
+			if (ReferencesAreKnown)
+			{
+				if (Cids.size() >= 1024)
+				{
+					GcCtx.AddRetainedCids(Cids);
+					Cids.clear();
+				}
+				continue;
+			}
+		}
+		StructuredItemsWithUnknownAttachments.push_back(Entry);
+	}
+
+	for (const auto& Entry : StructuredItemsWithUnknownAttachments)
+	{
+		BucketPayload&		Payload = Payloads[Entry.second];
+		const DiskLocation& Loc		= Payload.Location;
+		{
 			IoBuffer Buffer;
+			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 			{
-				RwLock::SharedLockScope __(m_IndexLock);
-				Stopwatch				Timer;
-				const auto				___ = MakeGuard([&] {
-					 uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
-					 WriteBlockTimeUs += ElapsedUs;
-					 WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
-				 });
-				if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
+				if (Buffer = GetStandaloneCacheValue(Loc.GetContentType(), Entry.first); !Buffer)
 				{
-					// We don't need to hold the index lock when we read a standalone file
-					__.ReleaseNow();
-					if (Buffer = GetStandaloneCacheValue(Loc, Key); !Buffer)
-					{
-						continue;
-					}
+					continue;
 				}
-				else if (Buffer = GetInlineCacheValue(Loc); !Buffer)
+			}
+			else
+			{
+				RwLock::SharedLockScope IndexLock(m_IndexLock);
+#if CALCULATE_BLOCKING_TIME
+				Stopwatch  Timer;
+				const auto ___ = MakeGuard([&] {
+					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
+					WriteBlockTimeUs += ElapsedUs;
+					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
+				});
+#endif	// CALCULATE_BLOCKING_TIME
+				if (auto It = m_Index.find(Entry.first); It != m_Index.end())
+				{
+					BucketPayload& UpdatePayload = m_Payloads[It->second];
+					Buffer						 = GetInlineCacheValue(UpdatePayload.Location);
+				}
+				if (!Buffer)
 				{
 					continue;
 				}
@@ -1345,7 +1446,39 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 			ZEN_ASSERT(Buffer);
 			ZEN_ASSERT(Buffer.GetContentType() == ZenContentType::kCbObject);
 			CbObject Obj(SharedBuffer{Buffer});
+			size_t	 CurrentCidCount = Cids.size();
 			Obj.IterateAttachments([&Cids](CbFieldView Field) { Cids.push_back(Field.AsAttachment()); });
+			if (m_EnableReferenceCaching)
+			{
+				RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
+#if CALCULATE_BLOCKING_TIME
+				Stopwatch  Timer;
+				const auto ___ = MakeGuard([&] {
+					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
+					ReadBlockTimeUs += ElapsedUs;
+					ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
+				});
+#endif	// CALCULATE_BLOCKING_TIME
+				if (auto It = m_Index.find(Entry.first); It != m_Index.end())
+				{
+					if (m_FirstReferenceIndex[It->second] == UnknownReferencesIndex)
+					{
+						SetReferences(IndexLock,
+									  m_FirstReferenceIndex[It->second],
+									  std::span<IoHash>(Cids.data() + CurrentCidCount, Cids.size() - CurrentCidCount));
+					}
+					else
+					{
+						Cids.resize(CurrentCidCount);
+						(void)GetReferences(IndexLock, m_FirstReferenceIndex[It->second], Cids);
+					}
+				}
+			}
+			if (Cids.size() >= 1024)
+			{
+				GcCtx.AddRetainedCids(Cids);
+				Cids.clear();
+			}
 		}
 	}
 
@@ -1434,6 +1567,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			// Clean up m_AccessTimes and m_Payloads vectors
 			std::vector<BucketPayload> Payloads;
 			std::vector<AccessTime>	   AccessTimes;
+			std::vector<size_t>		   FirstReferenceIndex;
 			IndexMap				   Index;
 
 			{
@@ -1447,6 +1581,10 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 				size_t					   EntryCount = m_Index.size();
 				Payloads.reserve(EntryCount);
 				AccessTimes.reserve(EntryCount);
+				if (m_EnableReferenceCaching)
+				{
+					FirstReferenceIndex.reserve(EntryCount);
+				}
 				Index.reserve(EntryCount);
 				for (auto It : m_Index)
 				{
@@ -1454,11 +1592,17 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 					size_t NewEntryIndex = Payloads.size();
 					Payloads.push_back(m_Payloads[OldEntryIndex]);
 					AccessTimes.push_back(m_AccessTimes[OldEntryIndex]);
+					if (m_EnableReferenceCaching)
+					{
+						FirstReferenceIndex.push_back(m_FirstReferenceIndex[It.second]);
+					}
 					Index.insert({It.first, NewEntryIndex});
 				}
 				m_Index.swap(Index);
 				m_Payloads.swap(Payloads);
 				m_AccessTimes.swap(AccessTimes);
+				m_FirstReferenceIndex.swap(FirstReferenceIndex);
+				CompactReferences(_);
 			}
 			GcCtx.AddDeletedCids(std::vector<IoHash>(DeletedChunks.begin(), DeletedChunks.end()));
 		}
@@ -1589,6 +1733,10 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 				size_t EntryIndex = m_Payloads.size();
 				m_Payloads.emplace_back(BucketPayload{.Location = RestoreLocation});
 				m_AccessTimes.emplace_back(GcClock::TickCount());
+				if (m_EnableReferenceCaching)
+				{
+					m_FirstReferenceIndex.emplace_back(UnknownReferencesIndex);
+				}
 				m_Index.insert({Key, EntryIndex});
 				m_TotalStandaloneSize.fetch_add(RestoreLocation.Size(), std::memory_order::relaxed);
 				DeletedChunks.erase(Key);
@@ -1757,8 +1905,9 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, size_t Index)
 	const BucketPayload& Payload = m_Payloads[Index];
 	if (Payload.Location.IsFlagSet(DiskLocation::kStructured))
 	{
-		IoBuffer Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile) ? GetStandaloneCacheValue(Payload.Location, Key)
-																				   : GetInlineCacheValue(Payload.Location);
+		IoBuffer Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile)
+							 ? GetStandaloneCacheValue(Payload.Location.GetContentType(), Key)
+							 : GetInlineCacheValue(Payload.Location);
 		CbObject Obj(SharedBuffer{Value});
 		Obj.IterateAttachments([&Attachments](CbFieldView Field) { Attachments.emplace_back(Field.AsAttachment()); });
 	}
@@ -1843,7 +1992,7 @@ ZenCacheDiskLayer::UpdateAccessTimes(const zen::access_tracking::AccessTimes& Ac
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::PutStandaloneCacheValue");
 
@@ -1957,13 +2106,18 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 
 	DiskLocation Loc(NewFileSize, EntryFlags);
 
-	RwLock::ExclusiveLockScope _(m_IndexLock);
+	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 	if (auto It = m_Index.find(HashKey); It == m_Index.end())
 	{
 		// Previously unknown object
 		size_t EntryIndex = m_Payloads.size();
 		m_Payloads.emplace_back(BucketPayload{.Location = Loc, .RawSize = Value.RawSize, .RawHash = Value.RawHash});
 		m_AccessTimes.emplace_back(GcClock::TickCount());
+		if (m_EnableReferenceCaching)
+		{
+			m_FirstReferenceIndex.emplace_back(UnknownReferencesIndex);
+			SetReferences(IndexLock, m_FirstReferenceIndex.back(), References);
+		}
 		m_Index.insert_or_assign(HashKey, EntryIndex);
 	}
 	else
@@ -1971,8 +2125,13 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 		// TODO: should check if write is idempotent and bail out if it is?
 		size_t EntryIndex = It.value();
 		ZEN_ASSERT_SLOW(EntryIndex < m_AccessTimes.size());
-		m_Payloads[EntryIndex] = BucketPayload{.Location = Loc, .RawSize = Value.RawSize, .RawHash = Value.RawHash};
-		m_AccessTimes.emplace_back(GcClock::TickCount());
+		BucketPayload& Payload = m_Payloads[EntryIndex];
+		Payload				   = BucketPayload{.Location = Loc, .RawSize = Value.RawSize, .RawHash = Value.RawHash};
+		if (m_EnableReferenceCaching)
+		{
+			SetReferences(IndexLock, m_FirstReferenceIndex[EntryIndex], References);
+		}
+		m_AccessTimes[EntryIndex] = GcClock::TickCount();
 		m_TotalStandaloneSize.fetch_sub(Loc.Size(), std::memory_order::relaxed);
 	}
 
@@ -1981,7 +2140,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::PutInlineCacheValue");
 
@@ -2000,7 +2159,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 		DiskLocation Location(BlockStoreLocation, m_PayloadAlignment, EntryFlags);
 		m_SlogFile.Append({.Key = HashKey, .Location = Location});
 
-		RwLock::ExclusiveLockScope _(m_IndexLock);
+		RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 		if (auto It = m_Index.find(HashKey); It != m_Index.end())
 		{
 			// TODO: should check if write is idempotent and bail out if it is?
@@ -2010,20 +2169,191 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 			ZEN_ASSERT_SLOW(EntryIndex < m_AccessTimes.size());
 			m_Payloads[EntryIndex]	  = (BucketPayload{.Location = Location, .RawSize = Value.RawSize, .RawHash = Value.RawHash});
 			m_AccessTimes[EntryIndex] = GcClock::TickCount();
+			if (m_EnableReferenceCaching)
+			{
+				SetReferences(IndexLock, m_FirstReferenceIndex[EntryIndex], References);
+			}
 		}
 		else
 		{
 			size_t EntryIndex = m_Payloads.size();
 			m_Payloads.emplace_back(BucketPayload{.Location = Location, .RawSize = Value.RawSize, .RawHash = Value.RawHash});
 			m_AccessTimes.emplace_back(GcClock::TickCount());
+			if (m_EnableReferenceCaching)
+			{
+				m_FirstReferenceIndex.emplace_back(UnknownReferencesIndex);
+				SetReferences(IndexLock, m_FirstReferenceIndex.back(), References);
+			}
 			m_Index.insert_or_assign(HashKey, EntryIndex);
 		}
 	});
 }
 
+void
+ZenCacheDiskLayer::CacheBucket::CompactReferences(RwLock::ExclusiveLockScope&)
+{
+	std::vector<size_t> FirstReferenceIndex;
+	std::vector<IoHash> NewReferenceHashes;
+	std::vector<size_t> NewNextReferenceHashesIndexes;
+
+	FirstReferenceIndex.reserve(m_ReferenceCount);
+	NewReferenceHashes.reserve(m_ReferenceCount);
+	NewNextReferenceHashesIndexes.reserve(m_ReferenceCount);
+
+	for (const auto& It : m_Index)
+	{
+		size_t SourceIndex = m_FirstReferenceIndex[It.second];
+		if (SourceIndex == UnknownReferencesIndex)
+		{
+			continue;
+		}
+		if (SourceIndex == NoReferencesIndex)
+		{
+			continue;
+		}
+		FirstReferenceIndex.push_back(NewNextReferenceHashesIndexes.size());
+		NewReferenceHashes.push_back(m_ReferenceHashes[SourceIndex]);
+		NewNextReferenceHashesIndexes.push_back(NoReferencesIndex);
+
+		SourceIndex = m_NextReferenceHashesIndexes[SourceIndex];
+		while (SourceIndex != NoReferencesIndex)
+		{
+			NewNextReferenceHashesIndexes.back() = NewReferenceHashes.size();
+			NewReferenceHashes.push_back(m_ReferenceHashes[SourceIndex]);
+			NewNextReferenceHashesIndexes.push_back(NoReferencesIndex);
+			SourceIndex = m_NextReferenceHashesIndexes[SourceIndex];
+		}
+	}
+	m_FirstReferenceIndex.swap(FirstReferenceIndex);
+	m_ReferenceHashes.swap(NewReferenceHashes);
+	m_NextReferenceHashesIndexes.swap(NewNextReferenceHashesIndexes);
+	m_ReferenceCount = m_ReferenceHashes.size();
+}
+
+size_t
+ZenCacheDiskLayer::CacheBucket::AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key)
+{
+	size_t ReferenceIndex = m_ReferenceHashes.size();
+	m_ReferenceHashes.push_back(Key);
+	m_NextReferenceHashesIndexes.push_back(NoReferencesIndex);
+	m_ReferenceCount++;
+	return ReferenceIndex;
+}
+
+void
+ZenCacheDiskLayer::CacheBucket::SetReferences(RwLock::ExclusiveLockScope& Lock,
+											  std::size_t&				  FirstReferenceIndex,
+											  std::span<IoHash>			  References)
+{
+	auto ReferenceIt = References.begin();
+
+	if (FirstReferenceIndex == UnknownReferencesIndex)
+	{
+		FirstReferenceIndex = NoReferencesIndex;
+	}
+
+	size_t CurrentIndex = FirstReferenceIndex;
+	if (CurrentIndex != NoReferencesIndex)
+	{
+		if (ReferenceIt != References.end())
+		{
+			ZEN_ASSERT_SLOW(*ReferenceIt != IoHash::Zero);
+			if (CurrentIndex == NoReferencesIndex)
+			{
+				CurrentIndex		= AllocateReferenceEntry(Lock, *ReferenceIt);
+				FirstReferenceIndex = CurrentIndex;
+			}
+			else
+			{
+				m_ReferenceHashes[CurrentIndex] = *ReferenceIt;
+			}
+			ReferenceIt++;
+		}
+	}
+	else
+	{
+		if (ReferenceIt != References.end())
+		{
+			ZEN_ASSERT_SLOW(*ReferenceIt != IoHash::Zero);
+			CurrentIndex = AllocateReferenceEntry(Lock, *ReferenceIt);
+			ReferenceIt++;
+		}
+		FirstReferenceIndex = CurrentIndex;
+	}
+
+	while (ReferenceIt != References.end())
+	{
+		ZEN_ASSERT(CurrentIndex != NoReferencesIndex);
+		ZEN_ASSERT_SLOW(*ReferenceIt != IoHash::Zero);
+		size_t ReferenceIndex = m_NextReferenceHashesIndexes[CurrentIndex];
+		if (ReferenceIndex == NoReferencesIndex)
+		{
+			ReferenceIndex							   = AllocateReferenceEntry(Lock, *ReferenceIt);
+			m_NextReferenceHashesIndexes[CurrentIndex] = ReferenceIndex;
+		}
+		else
+		{
+			m_ReferenceHashes[ReferenceIndex] = *ReferenceIt;
+		}
+		CurrentIndex = ReferenceIndex;
+		ReferenceIt++;
+	}
+
+	while (CurrentIndex != NoReferencesIndex)
+	{
+		size_t NextIndex = m_NextReferenceHashesIndexes[CurrentIndex];
+		if (NextIndex != NoReferencesIndex)
+		{
+			m_ReferenceHashes[CurrentIndex] = IoHash::Zero;
+			ZEN_ASSERT(m_ReferenceCount > 0);
+			m_ReferenceCount--;
+			m_NextReferenceHashesIndexes[CurrentIndex] = NoReferencesIndex;
+		}
+		CurrentIndex = NextIndex;
+	}
+}
+
+void
+ZenCacheDiskLayer::CacheBucket::RemoveReferences(RwLock::ExclusiveLockScope&, std::size_t& FirstReferenceIndex)
+{
+	if (FirstReferenceIndex == UnknownReferencesIndex)
+	{
+		return;
+	}
+	size_t CurrentIndex = FirstReferenceIndex;
+	while (CurrentIndex != NoReferencesIndex)
+	{
+		m_ReferenceHashes[CurrentIndex] = IoHash::Zero;
+		ZEN_ASSERT(m_ReferenceCount > 0);
+		m_ReferenceCount--;
+		CurrentIndex = m_NextReferenceHashesIndexes[CurrentIndex];
+	}
+	FirstReferenceIndex = UnknownReferencesIndex;
+}
+
+bool
+ZenCacheDiskLayer::CacheBucket::LockedGetReferences(std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
+{
+	if (FirstReferenceIndex == UnknownReferencesIndex)
+	{
+		return false;
+	}
+
+	size_t CurrentIndex = FirstReferenceIndex;
+	while (CurrentIndex != NoReferencesIndex)
+	{
+		ZEN_ASSERT_SLOW(m_ReferenceHashes[CurrentIndex] != IoHash::Zero);
+		OutReferences.push_back(m_ReferenceHashes[CurrentIndex]);
+		CurrentIndex = m_NextReferenceHashesIndexes[CurrentIndex];
+	}
+	return true;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
-ZenCacheDiskLayer::ZenCacheDiskLayer(const std::filesystem::path& RootDir) : m_RootDir(RootDir)
+ZenCacheDiskLayer::ZenCacheDiskLayer(const std::filesystem::path& RootDir, bool EnableReferenceCaching)
+: m_RootDir(RootDir)
+, m_EnableReferenceCaching(EnableReferenceCaching)
 {
 }
 
@@ -2060,7 +2390,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 		}
 		else
 		{
-			auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName));
+			auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName, m_EnableReferenceCaching));
 			Bucket			  = InsertResult.first->second.get();
 
 			std::filesystem::path BucketPath = m_RootDir;
@@ -2079,7 +2409,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 }
 
 void
-ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Put");
 
@@ -2109,7 +2439,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 		}
 		else
 		{
-			auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName));
+			auto InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName, m_EnableReferenceCaching));
 			Bucket			  = InsertResult.first->second.get();
 
 			std::filesystem::path BucketPath = m_RootDir;
@@ -2134,7 +2464,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 
 	ZEN_ASSERT(Bucket != nullptr);
 
-	Bucket->Put(HashKey, Value);
+	Bucket->Put(HashKey, Value, References);
 }
 
 void
@@ -2165,7 +2495,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 			continue;
 		}
 
-		auto		 InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName));
+		auto		 InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName, m_EnableReferenceCaching));
 		CacheBucket& Bucket		  = *InsertResult.first->second;
 
 		try
@@ -2352,7 +2682,8 @@ ZenCacheDiskLayer::Stats() const
 ZenCacheDiskLayer::Info
 ZenCacheDiskLayer::GetInfo() const
 {
-	ZenCacheDiskLayer::Info Info = {.Config = {.RootDir = m_RootDir}, .TotalSize = TotalSize()};
+	ZenCacheDiskLayer::Info Info = {.Config	   = {.RootDir = m_RootDir, .EnableReferenceCaching = m_EnableReferenceCaching},
+									.TotalSize = TotalSize()};
 	{
 		RwLock::SharedLockScope _(m_Lock);
 		Info.BucketNames.reserve(m_Buckets.size());
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index 62163100d..7e05430a2 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -93,6 +93,7 @@ public:
 	struct Configuration
 	{
 		std::filesystem::path RootDir;
+		bool				  EnableReferenceCaching;
 	};
 
 	struct BucketInfo
@@ -130,11 +131,11 @@ public:
 		std::vector<NamedBucketStats> BucketStats;
 	};
 
-	explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir);
+	explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir, bool EnableReferenceCaching);
 	~ZenCacheDiskLayer();
 
 	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
-	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 	bool Drop();
 	bool DropBucket(std::string_view Bucket);
 	void Flush();
@@ -160,12 +161,12 @@ private:
 	  */
 	struct CacheBucket
 	{
-		CacheBucket(std::string BucketName);
+		CacheBucket(std::string BucketName, bool EnableReferenceCaching);
 		~CacheBucket();
 
 		bool OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
 		bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
-		void Put(const IoHash& HashKey, const ZenCacheValue& Value);
+		void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 		bool Drop();
 		void Flush();
 		void ScrubStorage(ScrubContext& Ctx);
@@ -191,6 +192,7 @@ private:
 		Oid					  m_BucketId;
 		uint64_t			  m_LargeObjectThreshold = 128 * 1024;
 		std::atomic_bool	  m_IsFlushing{};
+		const bool			  m_EnableReferenceCaching = false;
 
 		// These files are used to manage storage of small objects for this bucket
 
@@ -199,6 +201,9 @@ private:
 
 #pragma pack(push)
 #pragma pack(1)
+		static const size_t UnknownReferencesIndex = (size_t)-1;
+		static const size_t NoReferencesIndex	   = (size_t)-2;
+
 		struct BucketPayload
 		{
 			DiskLocation Location;	// 12
@@ -218,16 +223,19 @@ private:
 		metrics::RequestStats m_GetOps;
 
 		mutable RwLock			   m_IndexLock;
+		IndexMap				   m_Index;
 		std::vector<AccessTime>	   m_AccessTimes;
 		std::vector<BucketPayload> m_Payloads;
-		IndexMap				   m_Index;
-
-		std::atomic_uint64_t m_TotalStandaloneSize{};
+		std::vector<size_t>		   m_FirstReferenceIndex;
+		std::vector<IoHash>		   m_ReferenceHashes;
+		std::vector<size_t>		   m_NextReferenceHashesIndexes;
+		size_t					   m_ReferenceCount = 0;
+		std::atomic_uint64_t	   m_TotalStandaloneSize{};
 
 		void	 BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
-		void	 PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
-		IoBuffer GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey) const;
-		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+		void	 PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
+		IoBuffer GetStandaloneCacheValue(ZenContentType ContentType, const IoHash& HashKey) const;
+		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 		IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const;
 		void	 MakeIndexSnapshot();
 		uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
@@ -236,13 +244,25 @@ private:
 		CbObject MakeManifest(IndexMap&& Index, std::vector<AccessTime>&& AccessTimes, const std::vector<BucketPayload>& Payloads);
 		void	 SaveManifest(CbObject&& Manifest);
 		CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, size_t Index) const;
+		void							CompactReferences(RwLock::ExclusiveLockScope&);
+		void		SetReferences(RwLock::ExclusiveLockScope&, std::size_t& FirstReferenceIndex, std::span<IoHash> References);
+		void		RemoveReferences(RwLock::ExclusiveLockScope&, std::size_t& FirstReferenceIndex);
+		inline bool GetReferences(RwLock::SharedLockScope&, std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
+		{
+			return LockedGetReferences(FirstReferenceIndex, OutReferences);
+		}
+		inline bool GetReferences(RwLock::ExclusiveLockScope&, std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
+		{
+			return LockedGetReferences(FirstReferenceIndex, OutReferences);
+		}
+		size_t AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key);
+		bool   LockedGetReferences(std::size_t FirstReferenceIndex, std::vector<IoHash>& OutReferences) const;
 		// These locks are here to avoid contention on file creation, therefore it's sufficient
 		// that we take the same lock for the same hash
 		//
 		// These locks are small and should really be spaced out so they don't share cache lines,
 		// but we don't currently access them at particularly high frequency so it should not be
 		// an issue in practice
-
 		mutable RwLock m_ShardedLocks[256];
 		inline RwLock& LockForHash(const IoHash& Hash) const { return m_ShardedLocks[Hash.Hash[19]]; }
 	};
@@ -251,6 +271,7 @@ private:
 	mutable RwLock												  m_Lock;
 	std::unordered_map<std::string, std::unique_ptr<CacheBucket>> m_Buckets;  // TODO: make this case insensitive
 	std::vector<std::unique_ptr<CacheBucket>>					  m_DroppedBuckets;
+	const bool													  m_EnableReferenceCaching;
 
 	ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete;
 	ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index f37fe1cc9..4ec7c56db 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -1051,7 +1051,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
 
 				if (Success && StoreLocal)
 				{
-					m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue);
+					m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {});
 					m_CacheStats.WriteCount++;
 				}
 			}
@@ -1064,59 +1064,65 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
 					AttachmentCount					 Count;
 					size_t							 NumAttachments = Package.GetAttachments().size();
 					std::vector<const CbAttachment*> AttachmentsToStoreLocally;
+					std::vector<IoHash>				 ReferencedAttachments;
 					AttachmentsToStoreLocally.reserve(NumAttachments);
 
-					CacheRecord.IterateAttachments(
-						[this, &Package, &Ref, &AttachmentsToStoreLocally, &Count, QueryLocal, StoreLocal, SkipData](CbFieldView HashView) {
-							IoHash Hash = HashView.AsHash();
-							if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
+					CacheRecord.IterateAttachments([this,
+													&Package,
+													&Ref,
+													&AttachmentsToStoreLocally,
+													&ReferencedAttachments,
+													&Count,
+													QueryLocal,
+													StoreLocal,
+													SkipData](CbFieldView HashView) {
+						IoHash Hash = HashView.AsHash();
+						ReferencedAttachments.push_back(Hash);
+						if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
+						{
+							if (Attachment->IsCompressedBinary())
 							{
-								if (Attachment->IsCompressedBinary())
+								if (StoreLocal)
 								{
-									if (StoreLocal)
-									{
-										AttachmentsToStoreLocally.emplace_back(Attachment);
-									}
-									Count.Valid++;
+									AttachmentsToStoreLocally.emplace_back(Attachment);
 								}
-								else
+								Count.Valid++;
+							}
+							else
+							{
+								ZEN_WARN("Uncompressed value '{}' from upstream cache record '{}/{}'",
+										 Hash,
+										 Ref.BucketSegment,
+										 Ref.HashKey);
+								Count.Invalid++;
+							}
+						}
+						else if (QueryLocal)
+						{
+							if (SkipData)
+							{
+								if (m_CidStore.ContainsChunk(Hash))
 								{
-									ZEN_WARN("Uncompressed value '{}' from upstream cache record '{}/{}'",
-											 Hash,
-											 Ref.BucketSegment,
-											 Ref.HashKey);
-									Count.Invalid++;
+									Count.Valid++;
 								}
 							}
-							else if (QueryLocal)
+							else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash))
 							{
-								if (SkipData)
+								CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
+								if (Compressed)
 								{
-									if (m_CidStore.ContainsChunk(Hash))
-									{
-										Count.Valid++;
-									}
+									Package.AddAttachment(CbAttachment(Compressed, Hash));
+									Count.Valid++;
 								}
-								else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash))
+								else
 								{
-									CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
-									if (Compressed)
-									{
-										Package.AddAttachment(CbAttachment(Compressed, Hash));
-										Count.Valid++;
-									}
-									else
-									{
-										ZEN_WARN("Uncompressed value '{}' stored in local cache '{}/{}'",
-												 Hash,
-												 Ref.BucketSegment,
-												 Ref.HashKey);
-										Count.Invalid++;
-									}
+									ZEN_WARN("Uncompressed value '{}' stored in local cache '{}/{}'", Hash, Ref.BucketSegment, Ref.HashKey);
+									Count.Invalid++;
 								}
 							}
-							Count.Total++;
-						});
+						}
+						Count.Total++;
+					});
 
 					if ((Count.Valid == Count.Total) || PartialRecord)
 					{
@@ -1126,7 +1132,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
 
 						if (StoreLocal)
 						{
-							m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
+							m_CacheStore
+								.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments);
 							m_CacheStats.WriteCount++;
 						}
 
@@ -1262,7 +1269,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 						 Ref.Namespace,
 						 Ref.BucketSegment,
 						 Ref.HashKey,
-						 {.Value = Body, .RawSize = RawSize, .RawHash = RawHash});
+						 {.Value = Body, .RawSize = RawSize, .RawHash = RawHash},
+						 {});
 		m_CacheStats.WriteCount++;
 
 		if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote))
@@ -1295,15 +1303,15 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 		}
 
 		Body.SetContentType(ZenContentType::kCbObject);
-		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
-		m_CacheStats.WriteCount++;
 
 		CbObjectView		CacheRecord(Body.Data());
 		std::vector<IoHash> ValidAttachments;
+		std::vector<IoHash> ReferencedAttachments;
 		int32_t				TotalCount = 0;
 
-		CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments](CbFieldView AttachmentHash) {
+		CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) {
 			const IoHash Hash = AttachmentHash.AsHash();
+			ReferencedAttachments.push_back(Hash);
 			if (m_CidStore.ContainsChunk(Hash))
 			{
 				ValidAttachments.emplace_back(Hash);
@@ -1311,6 +1319,9 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 			TotalCount++;
 		});
 
+		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}, ReferencedAttachments);
+		m_CacheStats.WriteCount++;
+
 		ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total) in {}",
 				  Ref.Namespace,
 				  Ref.BucketSegment,
@@ -1355,38 +1366,41 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 		AttachmentCount					 Count;
 		size_t							 NumAttachments = Package.GetAttachments().size();
 		std::vector<IoHash>				 ValidAttachments;
+		std::vector<IoHash>				 ReferencedAttachments;
 		std::vector<const CbAttachment*> AttachmentsToStoreLocally;
 		ValidAttachments.reserve(NumAttachments);
 		AttachmentsToStoreLocally.reserve(NumAttachments);
 
-		CacheRecord.IterateAttachments([this, &Ref, &Package, &AttachmentsToStoreLocally, &ValidAttachments, &Count](CbFieldView HashView) {
-			const IoHash Hash = HashView.AsHash();
-			if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
-			{
-				if (Attachment->IsCompressedBinary())
+		CacheRecord.IterateAttachments(
+			[this, &Ref, &Package, &AttachmentsToStoreLocally, &ValidAttachments, &ReferencedAttachments, &Count](CbFieldView HashView) {
+				const IoHash Hash = HashView.AsHash();
+				ReferencedAttachments.push_back(Hash);
+				if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
 				{
-					AttachmentsToStoreLocally.emplace_back(Attachment);
-					ValidAttachments.emplace_back(Hash);
-					Count.Valid++;
+					if (Attachment->IsCompressedBinary())
+					{
+						AttachmentsToStoreLocally.emplace_back(Attachment);
+						ValidAttachments.emplace_back(Hash);
+						Count.Valid++;
+					}
+					else
+					{
+						ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
+								 Ref.Namespace,
+								 Ref.BucketSegment,
+								 Ref.HashKey,
+								 ToString(HttpContentType::kCbPackage),
+								 Hash);
+						Count.Invalid++;
+					}
 				}
-				else
+				else if (m_CidStore.ContainsChunk(Hash))
 				{
-					ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
-							 Ref.Namespace,
-							 Ref.BucketSegment,
-							 Ref.HashKey,
-							 ToString(HttpContentType::kCbPackage),
-							 Hash);
-					Count.Invalid++;
+					ValidAttachments.emplace_back(Hash);
+					Count.Valid++;
 				}
-			}
-			else if (m_CidStore.ContainsChunk(Hash))
-			{
-				ValidAttachments.emplace_back(Hash);
-				Count.Valid++;
-			}
-			Count.Total++;
-		});
+				Count.Total++;
+			});
 
 		if (Count.Invalid > 0)
 		{
@@ -1397,7 +1411,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 		ZenCacheValue CacheValue;
 		CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer();
 		CacheValue.Value.SetContentType(ZenContentType::kCbObject);
-		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
+		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments);
 		m_CacheStats.WriteCount++;
 
 		for (const CbAttachment* Attachment : AttachmentsToStoreLocally)
@@ -1903,6 +1917,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
 	AttachmentCount					 Count;
 	size_t							 NumAttachments = Package->GetAttachments().size();
 	std::vector<IoHash>				 ValidAttachments;
+	std::vector<IoHash>				 ReferencedAttachments;
 	std::vector<const CbAttachment*> AttachmentsToStoreLocally;
 	ValidAttachments.reserve(NumAttachments);
 	AttachmentsToStoreLocally.reserve(NumAttachments);
@@ -1912,8 +1927,10 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
 	Stopwatch Timer;
 
 	Request.RecordObject.IterateAttachments(
-		[this, &Request, Package, &AttachmentsToStoreLocally, &ValidAttachments, &Count, &TransferredSize](CbFieldView HashView) {
+		[this, &Request, Package, &AttachmentsToStoreLocally, &ValidAttachments, &ReferencedAttachments, &Count, &TransferredSize](
+			CbFieldView HashView) {
 			const IoHash ValueHash = HashView.AsHash();
+			ReferencedAttachments.push_back(ValueHash);
 			if (const CbAttachment* Attachment = Package ? Package->FindAttachment(ValueHash) : nullptr)
 			{
 				if (Attachment->IsCompressedBinary())
@@ -1950,7 +1967,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
 	CacheValue.Value = IoBuffer(Record.GetSize());
 	Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize()));
 	CacheValue.Value.SetContentType(ZenContentType::kCbObject);
-	m_CacheStore.Put(Request.Context, Request.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue);
+	m_CacheStore.Put(Request.Context, Request.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue, ReferencedAttachments);
 	m_CacheStats.WriteCount++;
 
 	for (const CbAttachment* Attachment : AttachmentsToStoreLocally)
@@ -2216,7 +2233,13 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 					EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal) && AreDiskWritesAllowed();
 				if (StoreLocal)
 				{
-					m_CacheStore.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}});
+					std::vector<IoHash> ReferencedAttachments;
+					ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) {
+						const IoHash ValueHash = HashView.AsHash();
+						ReferencedAttachments.push_back(ValueHash);
+					});
+					m_CacheStore
+						.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}, ReferencedAttachments);
 					m_CacheStats.WriteCount++;
 				}
 				ParseValues(Request);
@@ -2407,7 +2430,8 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CacheRequestContext& C
 					{
 						RawSize = Chunk.DecodeRawSize();
 					}
-					m_CacheStore.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = Value, .RawSize = RawSize, .RawHash = RawHash});
+					m_CacheStore
+						.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = Value, .RawSize = RawSize, .RawHash = RawHash}, {});
 					m_CacheStats.WriteCount++;
 					TransferredSize = Chunk.GetCompressedSize();
 				}
@@ -2607,7 +2631,8 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(const CacheRequestContext& C
 											 *Namespace,
 											 Request.Key.Bucket,
 											 Request.Key.Hash,
-											 ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash});
+											 ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash},
+											 {});
 							m_CacheStats.WriteCount++;
 						}
 
@@ -2932,7 +2957,12 @@ HttpStructuredCacheService::GetLocalCacheRecords(const CacheRequestContext&
 				bool StoreLocal = EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed();
 				if (StoreLocal)
 				{
-					m_CacheStore.Put(Context, Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue});
+					std::vector<IoHash> ReferencedAttachments;
+					ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) {
+						const IoHash ValueHash = HashView.AsHash();
+						ReferencedAttachments.push_back(ValueHash);
+					});
+					m_CacheStore.Put(Context, Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}, ReferencedAttachments);
 					m_CacheStats.WriteCount++;
 				}
 			};
@@ -3122,7 +3152,8 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(const CacheRequestContext&
 									 Namespace,
 									 Key.Key.Bucket,
 									 Key.Key.Hash,
-									 {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash});
+									 {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash},
+									 {});
 					m_CacheStats.WriteCount++;
 				}
 			}
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index fe0b84f33..48463fcd8 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -61,13 +61,14 @@ IsKnownBadBucketName(std::string_view Bucket)
 ZenCacheNamespace::ZenCacheNamespace(GcManager&								  Gc,
 									 JobQueue&								  JobQueue,
 									 const std::filesystem::path&			  RootDir,
+									 bool									  EnableReferenceCaching,
 									 const ZenCacheMemoryLayer::Configuration MemLayerConfig)
 : GcStorage(Gc)
 , GcContributor(Gc)
 , m_RootDir(RootDir)
 , m_JobQueue(JobQueue)
 , m_MemLayer(m_JobQueue, MemLayerConfig)
-, m_DiskLayer(RootDir)
+, m_DiskLayer(RootDir, EnableReferenceCaching)
 {
 	ZEN_INFO("initializing structured cache at '{}'", RootDir);
 	CreateDirectories(RootDir);
@@ -116,7 +117,7 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 }
 
 void
-ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
 {
 	ZEN_TRACE_CPU("Z$::Namespace::Put");
 
@@ -126,7 +127,7 @@ ZenCacheNamespace::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 
 	ZEN_ASSERT(Value.Value.Size());
 
-	m_DiskLayer.Put(InBucket, HashKey, Value);
+	m_DiskLayer.Put(InBucket, HashKey, Value, References);
 
 	if (Value.Value.Size() <= m_DiskLayerSizeThreshold)
 	{
@@ -321,6 +322,7 @@ ZenCacheStore::ZenCacheStore(GcManager&				 Gc,
 			std::make_unique<ZenCacheNamespace>(Gc,
 												m_JobQueue,
 												m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, NamespaceName),
+												m_Configuration.EnableReferenceCaching,
 												m_Configuration.MemLayerConfig);
 	}
 }
@@ -476,7 +478,8 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 				   std::string_view			  Namespace,
 				   std::string_view			  Bucket,
 				   const IoHash&			  HashKey,
-				   const ZenCacheValue&		  Value)
+				   const ZenCacheValue&		  Value,
+				   std::span<IoHash>		  References)
 {
 	// Ad hoc rejection of known bad usage patterns for DDC bucket names
 
@@ -512,7 +515,7 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 
 	if (ZenCacheNamespace* Store = GetNamespace(Namespace); Store)
 	{
-		Store->Put(Bucket, HashKey, Value);
+		Store->Put(Bucket, HashKey, Value, References);
 		m_WriteCount++;
 		return;
 	}
@@ -625,6 +628,7 @@ ZenCacheStore::GetNamespace(std::string_view Namespace)
 		std::make_unique<ZenCacheNamespace>(m_Gc,
 											m_JobQueue,
 											m_Configuration.BasePath / fmt::format("{}{}", NamespaceDiskPrefix, Namespace),
+											m_Configuration.EnableReferenceCaching,
 											m_Configuration.MemLayerConfig));
 	return NewNamespace.first->second.get();
 }
@@ -803,7 +807,7 @@ TEST_CASE("z$.store")
 	GcManager Gc;
 
 	auto			  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", false);
 
 	const int kIterationCount = 100;
 
@@ -819,7 +823,7 @@ TEST_CASE("z$.store")
 		Value.Value = Obj.GetBuffer().AsIoBuffer();
 		Value.Value.SetContentType(ZenContentType::kCbObject);
 
-		Zcs.Put("test_bucket"sv, Key, Value);
+		Zcs.Put("test_bucket"sv, Key, Value, {});
 	}
 
 	for (int i = 0; i < kIterationCount; ++i)
@@ -859,7 +863,7 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", false);
 
 			CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() - 256);
 
@@ -869,7 +873,10 @@ TEST_CASE("z$.size")
 			for (size_t Key = 0; Key < Count; ++Key)
 			{
 				const size_t Bucket = Key % 4;
-				Zcs.Put(fmt::format("test_bucket-{}", Bucket), IoHash::HashBuffer(&Key, sizeof(uint32_t)), ZenCacheValue{.Value = Buffer});
+				Zcs.Put(fmt::format("test_bucket-{}", Bucket),
+						IoHash::HashBuffer(&Key, sizeof(uint32_t)),
+						ZenCacheValue{.Value = Buffer},
+						{});
 			}
 
 			CacheSize = Zcs.StorageSize();
@@ -879,7 +886,7 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", false);
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -902,7 +909,7 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", false);
 
 			CbObject CacheValue = CreateCacheValue(Zcs.DiskLayerThreshold() + 64);
 
@@ -912,7 +919,7 @@ TEST_CASE("z$.size")
 			for (size_t Key = 0; Key < Count; ++Key)
 			{
 				const size_t Bucket = Key % 4;
-				Zcs.Put(fmt::format("test_bucket-{}", Bucket), IoHash::HashBuffer(&Key, sizeof(uint32_t)), {.Value = Buffer});
+				Zcs.Put(fmt::format("test_bucket-{}", Bucket), IoHash::HashBuffer(&Key, sizeof(uint32_t)), {.Value = Buffer}, {});
 			}
 
 			CacheSize = Zcs.StorageSize();
@@ -922,7 +929,7 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", false);
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -961,7 +968,7 @@ TEST_CASE("z$.gc")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", true);
 			const auto		  Bucket = "teardrinker"sv;
 
 			// Create a cache record
@@ -978,7 +985,7 @@ TEST_CASE("z$.gc")
 			IoBuffer Buffer = Record.Save().GetBuffer().AsIoBuffer();
 			Buffer.SetContentType(ZenContentType::kCbObject);
 
-			Zcs.Put(Bucket, Key, {.Value = Buffer});
+			Zcs.Put(Bucket, Key, {.Value = Buffer}, Cids);
 
 			std::vector<IoHash> Keep;
 
@@ -998,7 +1005,7 @@ TEST_CASE("z$.gc")
 		// Expect timestamps to be serialized
 		{
 			GcManager			Gc;
-			ZenCacheNamespace	Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+			ZenCacheNamespace	Zcs(Gc, *JobQueue, TempDir.Path() / "cache", true);
 			std::vector<IoHash> Keep;
 
 			// Collect garbage with 1 hour max cache duration
@@ -1019,7 +1026,7 @@ TEST_CASE("z$.gc")
 	{
 		ScopedTemporaryDirectory TempDir;
 		GcManager				 Gc;
-		ZenCacheNamespace		 Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+		ZenCacheNamespace		 Zcs(Gc, *JobQueue, TempDir.Path() / "cache", true);
 		const auto				 Bucket		 = "fortysixandtwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -1028,7 +1035,7 @@ TEST_CASE("z$.gc")
 		for (const auto& Key : Keys)
 		{
 			IoBuffer Value = testutils::CreateBinaryCacheValue(128 << 10);
-			Zcs.Put(Bucket, Key, {.Value = Value});
+			Zcs.Put(Bucket, Key, {.Value = Value}, {});
 		}
 
 		{
@@ -1065,7 +1072,7 @@ TEST_CASE("z$.gc")
 		ScopedTemporaryDirectory TempDir;
 		GcManager				 Gc;
 		{
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", true);
 			const auto		  Bucket = "rightintwo"sv;
 
 			std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
@@ -1073,7 +1080,7 @@ TEST_CASE("z$.gc")
 			for (const auto& Key : Keys)
 			{
 				IoBuffer Value = testutils::CreateBinaryCacheValue(128);
-				Zcs.Put(Bucket, Key, {.Value = Value});
+				Zcs.Put(Bucket, Key, {.Value = Value}, {});
 			}
 
 			{
@@ -1107,7 +1114,7 @@ TEST_CASE("z$.gc")
 			}
 		}
 		{
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", true);
 			CHECK_EQ(0, Zcs.StorageSize().DiskSize);
 		}
 	}
@@ -1164,14 +1171,14 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 		WorkerThreadPool  ThreadPool(4);
 		GcManager		  Gc;
 		auto			  JobQueue = MakeJobQueue(1, "testqueue");
-		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path());
+		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path(), true);
 
 		{
 			std::atomic<size_t> WorkCompleted = 0;
 			for (const auto& Chunk : Chunks)
 			{
 				ThreadPool.ScheduleWork([&Zcs, &WorkCompleted, &Chunk]() {
-					Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer});
+					Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {});
 					WorkCompleted.fetch_add(1);
 				});
 			}
@@ -1232,7 +1239,7 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 			for (const auto& Chunk : NewChunks)
 			{
 				ThreadPool.ScheduleWork([&Zcs, &WorkCompleted, Chunk, &AddedChunkCount]() {
-					Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer});
+					Zcs.Put(Chunk.second.Bucket, Chunk.first, {.Value = Chunk.second.Buffer}, {});
 					AddedChunkCount.fetch_add(1);
 					WorkCompleted.fetch_add(1);
 				});
@@ -1395,14 +1402,14 @@ TEST_CASE("z$.namespaces")
 		Buffer.SetContentType(ZenContentType::kCbObject);
 
 		ZenCacheValue PutValue = {.Value = Buffer};
-		Zcs.Put(Context, ZenCacheStore::DefaultNamespace, Bucket, Key1, PutValue);
+		Zcs.Put(Context, ZenCacheStore::DefaultNamespace, Bucket, Key1, PutValue, {});
 
 		ZenCacheValue GetValue;
 		CHECK(Zcs.Get(Context, ZenCacheStore::DefaultNamespace, Bucket, Key1, GetValue));
 		CHECK(!Zcs.Get(Context, CustomNamespace, Bucket, Key1, GetValue));
 
 		// This should just be dropped as we don't allow creating of namespaces on the fly
-		Zcs.Put(Context, CustomNamespace, Bucket, Key1, PutValue);
+		Zcs.Put(Context, CustomNamespace, Bucket, Key1, PutValue, {});
 		CHECK(!Zcs.Get(Context, CustomNamespace, Bucket, Key1, GetValue));
 	}
 
@@ -1418,7 +1425,7 @@ TEST_CASE("z$.namespaces")
 		IoBuffer Buffer2 = CacheValue2.GetBuffer().AsIoBuffer();
 		Buffer2.SetContentType(ZenContentType::kCbObject);
 		ZenCacheValue PutValue2 = {.Value = Buffer2};
-		Zcs.Put(Context, CustomNamespace, Bucket, Key2, PutValue2);
+		Zcs.Put(Context, CustomNamespace, Bucket, Key2, PutValue2, {});
 
 		ZenCacheValue GetValue;
 		CHECK(!Zcs.Get(Context, ZenCacheStore::DefaultNamespace, Bucket, Key2, GetValue));
@@ -1460,7 +1467,7 @@ TEST_CASE("z$.drop.bucket")
 		Buffer.SetContentType(ZenContentType::kCbObject);
 
 		ZenCacheValue PutValue = {.Value = Buffer};
-		Zcs.Put(Context, Namespace, Bucket, Key, PutValue);
+		Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {});
 		return Key;
 	};
 	auto GetValue = [&Context](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) {
@@ -1533,7 +1540,7 @@ TEST_CASE("z$.drop.namespace")
 		Buffer.SetContentType(ZenContentType::kCbObject);
 
 		ZenCacheValue PutValue = {.Value = Buffer};
-		Zcs.Put(Context, Namespace, Bucket, Key, PutValue);
+		Zcs.Put(Context, Namespace, Bucket, Key, PutValue, {});
 		return Key;
 	};
 	auto GetValue = [&Context](ZenCacheStore& Zcs, std::string_view Namespace, std::string_view Bucket, const IoHash& Key) {
@@ -1610,7 +1617,7 @@ TEST_CASE("z$.blocked.disklayer.put")
 
 	GcManager		  Gc;
 	auto			  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", false);
 
 	CbObject CacheValue = CreateCacheValue(64 * 1024 + 64);
 
@@ -1619,7 +1626,7 @@ TEST_CASE("z$.blocked.disklayer.put")
 
 	size_t Key	   = Buffer.Size();
 	IoHash HashKey = IoHash::HashBuffer(&Key, sizeof(uint32_t));
-	Zcs.Put("test_bucket", HashKey, {.Value = Buffer});
+	Zcs.Put("test_bucket", HashKey, {.Value = Buffer}, {});
 
 	ZenCacheValue BufferGet;
 	CHECK(Zcs.Get("test_bucket", HashKey, BufferGet));
@@ -1629,7 +1636,7 @@ TEST_CASE("z$.blocked.disklayer.put")
 	Buffer2.SetContentType(ZenContentType::kCbObject);
 
 	// We should be able to overwrite even if the file is open for read
-	Zcs.Put("test_bucket", HashKey, {.Value = Buffer2});
+	Zcs.Put("test_bucket", HashKey, {.Value = Buffer2}, {});
 
 	MemoryView OldView = BufferGet.Value.GetView();
 
@@ -1705,7 +1712,7 @@ TEST_CASE("z$.scrub")
 	GcManager			  Gc;
 	CidStore			  CidStore(Gc);
 	auto				  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace	  Zcs(Gc, *JobQueue, TempDir.Path() / "cache");
+	ZenCacheNamespace	  Zcs(Gc, *JobQueue, TempDir.Path() / "cache", true);
 	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
 	CidStore.Initialize(CidConfig);
 
@@ -1713,12 +1720,14 @@ TEST_CASE("z$.scrub")
 		[&](bool IsStructured, std::string_view BucketName, const std::vector<IoHash>& Cids, const std::vector<size_t>& AttachmentSizes) {
 			for (const IoHash& Cid : Cids)
 			{
-				CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes);
-				Zcs.Put("mybucket", Cid, {.Value = Record.Record});
+				CacheRecord			Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes);
+				std::vector<IoHash> AttachmentHashes;
 				for (const CompressedBuffer& Attachment : Record.Attachments)
 				{
-					CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), Attachment.DecodeRawHash());
+					AttachmentHashes.push_back(Attachment.DecodeRawHash());
+					CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back());
 				}
+				Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes);
 			}
 		};
 
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index eca2f1880..672858fa0 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -81,11 +81,12 @@ public:
 	ZenCacheNamespace(GcManager&							   Gc,
 					  JobQueue&								   JobQueue,
 					  const std::filesystem::path&			   RootDir,
+					  bool									   EnableReferenceCaching,
 					  const ZenCacheMemoryLayer::Configuration MemLayerConfig = {});
 	~ZenCacheNamespace();
 
 	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
-	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
+	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 
 	bool DropBucket(std::string_view Bucket);
 	void EnumerateBucketContents(std::string_view																		 Bucket,
@@ -144,6 +145,7 @@ public:
 	{
 		std::filesystem::path			   BasePath;
 		bool							   AllowAutomaticCreationOfNamespaces = false;
+		bool							   EnableReferenceCaching			  = false;
 		ZenCacheMemoryLayer::Configuration MemLayerConfig;
 		struct LogConfig
 		{
@@ -191,7 +193,8 @@ public:
 			 std::string_view			Namespace,
 			 std::string_view			Bucket,
 			 const IoHash&				HashKey,
-			 const ZenCacheValue&		Value);
+			 const ZenCacheValue&		Value,
+			 std::span<IoHash>			References);
 	bool DropBucket(std::string_view Namespace, std::string_view Bucket);
 	bool DropNamespace(std::string_view Namespace);
 	void Flush();
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 435b66a83..342f41b68 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -802,6 +802,9 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled);
 	LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log");
 	LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log");
+	LuaOptions.AddOption("cache.referencecache"sv,
+						 ServerOptions.StructuredCacheConfig.EnableReferenceCaching,
+						 "cache-reference-cache-enabled");
 
 	LuaOptions.AddOption("cache.memlayer.targetfootprint"sv,
 						 ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes,
@@ -1228,6 +1231,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 
 	options.add_option("cache",
 					   "",
+					   "cache-reference-cache-enabled",
+					   "Whether caching of references is enabled",
+					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.EnableReferenceCaching)->default_value("false"),
+					   "");
+
+	options.add_option("cache",
+					   "",
 					   "cache-memlayer-targetfootprint",
 					   "Max allowed memory used by cache memory layer per namespace in bytes. Default set to 536870912 (512 Mb).",
 					   cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes)->default_value("536870912"),
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 3e80ba10f..ee57b23cc 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -123,6 +123,7 @@ struct ZenStructuredCacheConfig
 	bool	 Enabled				 = true;
 	bool	 WriteLogEnabled		 = false;
 	bool	 AccessLogEnabled		 = false;
+	bool	 EnableReferenceCaching	 = false;
 	uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024;
 	uint64_t MemTrimIntervalSeconds	 = 60;
 	uint64_t MemMaxAgeSeconds		 = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count());
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index e000976ea..02760f6dd 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -507,14 +507,17 @@ ProjectStore::Oplog::GatherReferences(GcContext& GcCtx)
 		return;
 	}
 
-	tsl::robin_set<IoHash> AttachmentHashes;
+	std::vector<IoHash> Cids;
+	Cids.reserve(1024);
 	IterateOplog([&](CbObject Op) {
-		Op.IterateAttachments([&](CbFieldView Visitor) {
-			IoHash Attachment = Visitor.AsAttachment();
-			AttachmentHashes.insert(Attachment);
-		});
+		Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); });
+		if (Cids.size() >= 1024)
+		{
+			GcCtx.AddRetainedCids(Cids);
+			Cids.clear();
+		}
 	});
-	GcCtx.AddRetainedCids(std::vector<IoHash>(AttachmentHashes.begin(), AttachmentHashes.end()));
+	GcCtx.AddRetainedCids(Cids);
 }
 
 uint64_t
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index e97d79cb5..1f5c91a58 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -438,6 +438,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 		*m_JobQueue,
 		ZenCacheStore::Configuration{.BasePath							 = m_DataRoot / "cache",
 									 .AllowAutomaticCreationOfNamespaces = true,
+									 .EnableReferenceCaching			 = ServerOptions.StructuredCacheConfig.EnableReferenceCaching,
 									 .MemLayerConfig = {.TargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes,
 														.TrimIntervalSeconds  = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds,
 														.MaxAgeSeconds		  = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds},