Relax constraint on partial cache records.

1 files changed, 60 insertions, 72 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index d3da98620..ec27265a7 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -454,52 +454,39 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
 
 		CbObjectView		CacheRecord(Body.Data());
 		std::vector<IoHash> ValidAttachments;
-		uint32_t			AttachmentCount = 0;
+		int32_t				TotalCount = 0;
 
-		CacheRecord.IterateAttachments([this, &AttachmentCount, &ValidAttachments](CbFieldView AttachmentHash) {
+		CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments](CbFieldView AttachmentHash) {
 			const IoHash Hash = AttachmentHash.AsHash();
 			if (m_CidStore.ContainsChunk(Hash))
 			{
 				ValidAttachments.emplace_back(Hash);
 			}
-			AttachmentCount++;
+			TotalCount++;
 		});
 
-		const uint32_t ValidCount		= static_cast<uint32_t>(ValidAttachments.size());
-		const bool	   ValidCacheRecord = ValidCount == AttachmentCount;
+		const bool IsPartialCacheRecord = TotalCount != static_cast<int32_t>(ValidAttachments.size());
 
-		if (ValidCacheRecord)
-		{
-			ZEN_DEBUG("PUT - '{}/{}' {} '{}', {} attachments",
-					  Ref.BucketSegment,
-					  Ref.HashKey,
-					  NiceBytes(Body.Size()),
-					  ToString(ContentType),
-					  ValidCount);
-
-			m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body});
+		ZEN_DEBUG("PUT - '{}/{}' {} '{}', {} attachments {}",
+				  Ref.BucketSegment,
+				  Ref.HashKey,
+				  NiceBytes(Body.Size()),
+				  ToString(ContentType),
+				  ValidAttachments.size(),
+				  IsPartialCacheRecord ? "(PARTIAL)"sv : ""sv);
 
-			if (StoreUpstream)
-			{
-				ZEN_ASSERT(m_UpstreamCache);
-				auto Result = m_UpstreamCache->EnqueueUpstream({.Type		= ZenContentType::kCbObject,
-																.CacheKey	= {Ref.BucketSegment, Ref.HashKey},
-																.PayloadIds = std::move(ValidAttachments)});
-			}
+		Body.SetContentType(ZenContentType::kCbObject);
+		m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body});
 
-			Request.WriteResponse(HttpResponseCode::Created);
-		}
-		else
+		if (StoreUpstream && !IsPartialCacheRecord)
 		{
-			ZEN_WARN("PUT - '{}/{}' '{}' FAILED, found {}/{} attachments",
-					 Ref.BucketSegment,
-					 Ref.HashKey,
-					 ToString(ContentType),
-					 ValidCount,
-					 AttachmentCount);
-
-			Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Missing attachments"sv);
+			ZEN_ASSERT(m_UpstreamCache);
+			auto Result = m_UpstreamCache->EnqueueUpstream({.Type		= ZenContentType::kCbObject,
+															.CacheKey	= {Ref.BucketSegment, Ref.HashKey},
+															.PayloadIds = std::move(ValidAttachments)});
 		}
+
+		Request.WriteResponse(HttpResponseCode::Created);
 	}
 	else if (ContentType == HttpContentType::kCbPackage)
 	{
@@ -511,62 +498,63 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
 			return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv);
 		}
 
-		CbObject CacheRecord = Package.GetObject();
-
-		std::span<const CbAttachment> Attachments = Package.GetAttachments();
-		std::vector<IoHash>			  ValidAttachments;
-		int32_t						  NewAttachmentCount = 0;
+		CbObject			CacheRecord = Package.GetObject();
+		std::vector<IoHash> ValidAttachments;
+		int32_t				TotalCount	 = 0;
+		int32_t				NewCount	 = 0;
+		int32_t				InvalidCount = 0;
 
-		ValidAttachments.reserve(Attachments.size());
+		ValidAttachments.reserve(Package.GetAttachments().size());
 
-		CacheRecord.IterateAttachments([this, &Ref, &Package, &ValidAttachments, &NewAttachmentCount](CbFieldView AttachmentHash) {
-			if (const CbAttachment* Attachment = Package.FindAttachment(AttachmentHash.AsHash()))
-			{
-				if (Attachment->IsCompressedBinary())
+		CacheRecord.IterateAttachments(
+			[this, &Ref, &Package, &ValidAttachments, &TotalCount, &NewCount, &InvalidCount](CbFieldView HashView) {
+				const IoHash Hash = HashView.AsHash();
+				if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
 				{
-					CompressedBuffer	   Chunk		= Attachment->AsCompressedBinary();
-					CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk);
+					if (Attachment->IsCompressedBinary())
+					{
+						CompressedBuffer	   Chunk		= Attachment->AsCompressedBinary();
+						CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk);
 
-					ValidAttachments.emplace_back(InsertResult.DecompressedId);
+						ValidAttachments.emplace_back(InsertResult.DecompressedId);
 
-					if (InsertResult.New)
+						if (InsertResult.New)
+						{
+							NewCount++;
+						}
+					}
+					else
 					{
-						NewAttachmentCount++;
+						ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed",
+								 Ref.BucketSegment,
+								 Ref.HashKey,
+								 ToString(HttpContentType::kCbPackage),
+								 Hash);
+						InvalidCount++;
 					}
 				}
-				else
+				else if (m_CidStore.ContainsChunk(Hash))
 				{
-					ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed",
-							 Ref.BucketSegment,
-							 Ref.HashKey,
-							 ToString(HttpContentType::kCbPackage),
-							 AttachmentHash.AsHash());
+					ValidAttachments.emplace_back(Hash);
 				}
-			}
-			else
-			{
-				ZEN_WARN("PUT - '{}/{}' '{}' FAILED, missing attachment '{}'",
-						 Ref.BucketSegment,
-						 Ref.HashKey,
-						 ToString(HttpContentType::kCbPackage),
-						 AttachmentHash.AsHash());
-			}
-		});
+				TotalCount++;
+			});
 
-		const bool IsPartialCacheRecord = ValidAttachments.size() != Attachments.size();
+		if (InvalidCount > 0)
+		{
+			return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv);
+		}
 
-		// if (!AttachmentsValid)
-		//{
-		//	return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachments"sv);
-		//}
+		const bool IsPartialCacheRecord = TotalCount != static_cast<int32_t>(ValidAttachments.size());
 
-		ZEN_DEBUG("PUT - '{}/{}' {} '{}', {}/{} new attachments",
+		ZEN_DEBUG("PUT - '{}/{}' {} '{}', {}/{} new attachments {}",
 				  Ref.BucketSegment,
 				  Ref.HashKey,
 				  NiceBytes(Body.GetSize()),
 				  ToString(ContentType),
-				  NewAttachmentCount,
-				  Attachments.size());
+				  NewCount,
+				  TotalCount,
+				  IsPartialCacheRecord ? "(PARTIAL)"sv : ""sv);
 
 		IoBuffer CacheRecordValue = CacheRecord.GetBuffer().AsIoBuffer();
 		CacheRecordValue.SetContentType(ZenContentType::kCbObject);