cache reference tracking (#455)

- Feature: Add caching of referenced CId content for structured cache records, this avoid disk thrashing when gathering references for GC
  - disabled by default, enable with `--cache-reference-cache-enabled`
- Improvement: Faster collection of referenced CId content in project store

1 files changed, 108 insertions, 77 deletions

diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index f37fe1cc9..4ec7c56db 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -1051,7 +1051,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
 
 				if (Success && StoreLocal)
 				{
-					m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue);
+					m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {});
 					m_CacheStats.WriteCount++;
 				}
 			}
@@ -1064,59 +1064,65 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
 					AttachmentCount					 Count;
 					size_t							 NumAttachments = Package.GetAttachments().size();
 					std::vector<const CbAttachment*> AttachmentsToStoreLocally;
+					std::vector<IoHash>				 ReferencedAttachments;
 					AttachmentsToStoreLocally.reserve(NumAttachments);
 
-					CacheRecord.IterateAttachments(
-						[this, &Package, &Ref, &AttachmentsToStoreLocally, &Count, QueryLocal, StoreLocal, SkipData](CbFieldView HashView) {
-							IoHash Hash = HashView.AsHash();
-							if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
+					CacheRecord.IterateAttachments([this,
+													&Package,
+													&Ref,
+													&AttachmentsToStoreLocally,
+													&ReferencedAttachments,
+													&Count,
+													QueryLocal,
+													StoreLocal,
+													SkipData](CbFieldView HashView) {
+						IoHash Hash = HashView.AsHash();
+						ReferencedAttachments.push_back(Hash);
+						if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
+						{
+							if (Attachment->IsCompressedBinary())
 							{
-								if (Attachment->IsCompressedBinary())
+								if (StoreLocal)
 								{
-									if (StoreLocal)
-									{
-										AttachmentsToStoreLocally.emplace_back(Attachment);
-									}
-									Count.Valid++;
+									AttachmentsToStoreLocally.emplace_back(Attachment);
 								}
-								else
+								Count.Valid++;
+							}
+							else
+							{
+								ZEN_WARN("Uncompressed value '{}' from upstream cache record '{}/{}'",
+										 Hash,
+										 Ref.BucketSegment,
+										 Ref.HashKey);
+								Count.Invalid++;
+							}
+						}
+						else if (QueryLocal)
+						{
+							if (SkipData)
+							{
+								if (m_CidStore.ContainsChunk(Hash))
 								{
-									ZEN_WARN("Uncompressed value '{}' from upstream cache record '{}/{}'",
-											 Hash,
-											 Ref.BucketSegment,
-											 Ref.HashKey);
-									Count.Invalid++;
+									Count.Valid++;
 								}
 							}
-							else if (QueryLocal)
+							else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash))
 							{
-								if (SkipData)
+								CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
+								if (Compressed)
 								{
-									if (m_CidStore.ContainsChunk(Hash))
-									{
-										Count.Valid++;
-									}
+									Package.AddAttachment(CbAttachment(Compressed, Hash));
+									Count.Valid++;
 								}
-								else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash))
+								else
 								{
-									CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
-									if (Compressed)
-									{
-										Package.AddAttachment(CbAttachment(Compressed, Hash));
-										Count.Valid++;
-									}
-									else
-									{
-										ZEN_WARN("Uncompressed value '{}' stored in local cache '{}/{}'",
-												 Hash,
-												 Ref.BucketSegment,
-												 Ref.HashKey);
-										Count.Invalid++;
-									}
+									ZEN_WARN("Uncompressed value '{}' stored in local cache '{}/{}'", Hash, Ref.BucketSegment, Ref.HashKey);
+									Count.Invalid++;
 								}
 							}
-							Count.Total++;
-						});
+						}
+						Count.Total++;
+					});
 
 					if ((Count.Valid == Count.Total) || PartialRecord)
 					{
@@ -1126,7 +1132,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
 
 						if (StoreLocal)
 						{
-							m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
+							m_CacheStore
+								.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments);
 							m_CacheStats.WriteCount++;
 						}
 
@@ -1262,7 +1269,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 						 Ref.Namespace,
 						 Ref.BucketSegment,
 						 Ref.HashKey,
-						 {.Value = Body, .RawSize = RawSize, .RawHash = RawHash});
+						 {.Value = Body, .RawSize = RawSize, .RawHash = RawHash},
+						 {});
 		m_CacheStats.WriteCount++;
 
 		if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote))
@@ -1295,15 +1303,15 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 		}
 
 		Body.SetContentType(ZenContentType::kCbObject);
-		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
-		m_CacheStats.WriteCount++;
 
 		CbObjectView		CacheRecord(Body.Data());
 		std::vector<IoHash> ValidAttachments;
+		std::vector<IoHash> ReferencedAttachments;
 		int32_t				TotalCount = 0;
 
-		CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments](CbFieldView AttachmentHash) {
+		CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) {
 			const IoHash Hash = AttachmentHash.AsHash();
+			ReferencedAttachments.push_back(Hash);
 			if (m_CidStore.ContainsChunk(Hash))
 			{
 				ValidAttachments.emplace_back(Hash);
@@ -1311,6 +1319,9 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 			TotalCount++;
 		});
 
+		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}, ReferencedAttachments);
+		m_CacheStats.WriteCount++;
+
 		ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total) in {}",
 				  Ref.Namespace,
 				  Ref.BucketSegment,
@@ -1355,38 +1366,41 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 		AttachmentCount					 Count;
 		size_t							 NumAttachments = Package.GetAttachments().size();
 		std::vector<IoHash>				 ValidAttachments;
+		std::vector<IoHash>				 ReferencedAttachments;
 		std::vector<const CbAttachment*> AttachmentsToStoreLocally;
 		ValidAttachments.reserve(NumAttachments);
 		AttachmentsToStoreLocally.reserve(NumAttachments);
 
-		CacheRecord.IterateAttachments([this, &Ref, &Package, &AttachmentsToStoreLocally, &ValidAttachments, &Count](CbFieldView HashView) {
-			const IoHash Hash = HashView.AsHash();
-			if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
-			{
-				if (Attachment->IsCompressedBinary())
+		CacheRecord.IterateAttachments(
+			[this, &Ref, &Package, &AttachmentsToStoreLocally, &ValidAttachments, &ReferencedAttachments, &Count](CbFieldView HashView) {
+				const IoHash Hash = HashView.AsHash();
+				ReferencedAttachments.push_back(Hash);
+				if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
 				{
-					AttachmentsToStoreLocally.emplace_back(Attachment);
-					ValidAttachments.emplace_back(Hash);
-					Count.Valid++;
+					if (Attachment->IsCompressedBinary())
+					{
+						AttachmentsToStoreLocally.emplace_back(Attachment);
+						ValidAttachments.emplace_back(Hash);
+						Count.Valid++;
+					}
+					else
+					{
+						ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
+								 Ref.Namespace,
+								 Ref.BucketSegment,
+								 Ref.HashKey,
+								 ToString(HttpContentType::kCbPackage),
+								 Hash);
+						Count.Invalid++;
+					}
 				}
-				else
+				else if (m_CidStore.ContainsChunk(Hash))
 				{
-					ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
-							 Ref.Namespace,
-							 Ref.BucketSegment,
-							 Ref.HashKey,
-							 ToString(HttpContentType::kCbPackage),
-							 Hash);
-					Count.Invalid++;
+					ValidAttachments.emplace_back(Hash);
+					Count.Valid++;
 				}
-			}
-			else if (m_CidStore.ContainsChunk(Hash))
-			{
-				ValidAttachments.emplace_back(Hash);
-				Count.Valid++;
-			}
-			Count.Total++;
-		});
+				Count.Total++;
+			});
 
 		if (Count.Invalid > 0)
 		{
@@ -1397,7 +1411,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 		ZenCacheValue CacheValue;
 		CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer();
 		CacheValue.Value.SetContentType(ZenContentType::kCbObject);
-		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue);
+		m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments);
 		m_CacheStats.WriteCount++;
 
 		for (const CbAttachment* Attachment : AttachmentsToStoreLocally)
@@ -1903,6 +1917,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
 	AttachmentCount					 Count;
 	size_t							 NumAttachments = Package->GetAttachments().size();
 	std::vector<IoHash>				 ValidAttachments;
+	std::vector<IoHash>				 ReferencedAttachments;
 	std::vector<const CbAttachment*> AttachmentsToStoreLocally;
 	ValidAttachments.reserve(NumAttachments);
 	AttachmentsToStoreLocally.reserve(NumAttachments);
@@ -1912,8 +1927,10 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
 	Stopwatch Timer;
 
 	Request.RecordObject.IterateAttachments(
-		[this, &Request, Package, &AttachmentsToStoreLocally, &ValidAttachments, &Count, &TransferredSize](CbFieldView HashView) {
+		[this, &Request, Package, &AttachmentsToStoreLocally, &ValidAttachments, &ReferencedAttachments, &Count, &TransferredSize](
+			CbFieldView HashView) {
 			const IoHash ValueHash = HashView.AsHash();
+			ReferencedAttachments.push_back(ValueHash);
 			if (const CbAttachment* Attachment = Package ? Package->FindAttachment(ValueHash) : nullptr)
 			{
 				if (Attachment->IsCompressedBinary())
@@ -1950,7 +1967,7 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
 	CacheValue.Value = IoBuffer(Record.GetSize());
 	Record.CopyTo(MutableMemoryView(CacheValue.Value.MutableData(), CacheValue.Value.GetSize()));
 	CacheValue.Value.SetContentType(ZenContentType::kCbObject);
-	m_CacheStore.Put(Request.Context, Request.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue);
+	m_CacheStore.Put(Request.Context, Request.Namespace, Request.Key.Bucket, Request.Key.Hash, CacheValue, ReferencedAttachments);
 	m_CacheStats.WriteCount++;
 
 	for (const CbAttachment* Attachment : AttachmentsToStoreLocally)
@@ -2216,7 +2233,13 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(const CacheRequestContext&
 					EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal) && AreDiskWritesAllowed();
 				if (StoreLocal)
 				{
-					m_CacheStore.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}});
+					std::vector<IoHash> ReferencedAttachments;
+					ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) {
+						const IoHash ValueHash = HashView.AsHash();
+						ReferencedAttachments.push_back(ValueHash);
+					});
+					m_CacheStore
+						.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = {Request.RecordCacheValue}}, ReferencedAttachments);
 					m_CacheStats.WriteCount++;
 				}
 				ParseValues(Request);
@@ -2407,7 +2430,8 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CacheRequestContext& C
 					{
 						RawSize = Chunk.DecodeRawSize();
 					}
-					m_CacheStore.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = Value, .RawSize = RawSize, .RawHash = RawHash});
+					m_CacheStore
+						.Put(Context, *Namespace, Key.Bucket, Key.Hash, {.Value = Value, .RawSize = RawSize, .RawHash = RawHash}, {});
 					m_CacheStats.WriteCount++;
 					TransferredSize = Chunk.GetCompressedSize();
 				}
@@ -2607,7 +2631,8 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(const CacheRequestContext& C
 											 *Namespace,
 											 Request.Key.Bucket,
 											 Request.Key.Hash,
-											 ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash});
+											 ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash},
+											 {});
 							m_CacheStats.WriteCount++;
 						}
 
@@ -2932,7 +2957,12 @@ HttpStructuredCacheService::GetLocalCacheRecords(const CacheRequestContext&
 				bool StoreLocal = EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed();
 				if (StoreLocal)
 				{
-					m_CacheStore.Put(Context, Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue});
+					std::vector<IoHash> ReferencedAttachments;
+					ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) {
+						const IoHash ValueHash = HashView.AsHash();
+						ReferencedAttachments.push_back(ValueHash);
+					});
+					m_CacheStore.Put(Context, Namespace, Key.Bucket, Key.Hash, {.Value = Record.CacheValue}, ReferencedAttachments);
 					m_CacheStats.WriteCount++;
 				}
 			};
@@ -3122,7 +3152,8 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(const CacheRequestContext&
 									 Namespace,
 									 Key.Key.Bucket,
 									 Key.Key.Hash,
-									 {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash});
+									 {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash},
+									 {});
 					m_CacheStats.WriteCount++;
 				}
 			}