GetCacheChunksRequest with separate policy and OptionalValueFilter

2 files changed, 126 insertions, 47 deletions

diff --git a/zenutil/cache/cacherequests.cpp b/zenutil/cache/cacherequests.cpp
index 21e7bb3b7..97a8ac606 100644
--- a/zenutil/cache/cacherequests.cpp
+++ b/zenutil/cache/cacherequests.cpp
@@ -441,11 +441,8 @@ namespace cacherequests {
 					Writer.BeginObject();
 					{
 						WriteCacheRequestKey(Writer, Key);
-						if (Policy.RecordPolicies.size() != 0)
-						{
-							const std::optional<CacheRecordPolicy>& RecordPolicy = Policy.RecordPolicies[Index];
-							WriteOptionalCacheRequestPolicy(Writer, "Policy"sv, Policy.DefaultPolicy, RecordPolicy);
-						}
+						const std::optional<CacheRecordPolicy>& RecordPolicy = Policy.RecordPolicies[Index];
+						WriteOptionalCacheRequestPolicy(Writer, "Policy"sv, Policy.DefaultPolicy, RecordPolicy);
 					}
 					Writer.EndObject();
 				}
@@ -850,7 +847,7 @@ namespace cacherequests {
 		return true;
 	}
 
-	bool GetCacheChunksRequest::Parse(const CbObjectView& BatchObject)
+	bool GetCacheChunksRequest::Parse(const CbObjectView& BatchObject, ChunksRequestPolicy& OutPolicy)
 	{
 		ZEN_ASSERT(BatchObject["Method"sv].AsString() == "GetCacheChunks"sv);
 
@@ -861,17 +858,19 @@ namespace cacherequests {
 			return false;
 		}
 
-		Namespace	  = *RequestNamespace;
-		DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy"sv).value_or(CachePolicy::Default);
+		Namespace				= *RequestNamespace;
+		OutPolicy.DefaultPolicy = GetCachePolicy(Params, "DefaultPolicy"sv).value_or(CachePolicy::Default);
 
 		CbArrayView RequestsArray = Params["ChunkRequests"sv].AsArrayView();
 		Requests.reserve(RequestsArray.Num());
+		OutPolicy.ChunkPolicies.reserve(RequestsArray.Num());
 		for (CbFieldView RequestField : RequestsArray)
 		{
 			CbObjectView RequestObject = RequestField.AsObjectView();
 			CbObjectView KeyObject	   = RequestObject["Key"sv].AsObjectView();
 
-			GetCacheChunkRequest& Request = Requests.emplace_back();
+			GetCacheChunkRequest&		Request		= Requests.emplace_back();
+			std::optional<CachePolicy>& ChunkPolicy = OutPolicy.ChunkPolicies.emplace_back();
 
 			if (!GetRequestCacheKey(KeyObject, Request.Key))
 			{
@@ -883,37 +882,66 @@ namespace cacherequests {
 			Request.RawOffset = RequestObject["RawOffset"sv].AsUInt64();
 			Request.RawSize	  = RequestObject["RawSize"sv].AsUInt64(UINT64_MAX);
 
-			Request.Policy = GetCachePolicy(RequestObject, "Policy");
+			ChunkPolicy = GetCachePolicy(RequestObject, "Policy");
 		}
 		return true;
 	}
 
-	bool GetCacheChunksRequest::Format(CbPackage& OutPackage) const
+	bool GetCacheChunksRequest::Format(CbPackage&					 OutPackage,
+									   const ChunksRequestPolicy&	 Policy,
+									   const std::span<const size_t> OptionalValueFilter) const
 	{
+		ZEN_ASSERT(Requests.size() == Policy.ChunkPolicies.size());
 		CbObjectWriter Writer;
 		Writer << "Method"sv
 			   << "GetCacheChunks"sv;
 
 		Writer.BeginObject("Params"sv);
 		{
-			Writer << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy);
+			Writer << "DefaultPolicy"sv << WriteToString<128>(Policy.DefaultPolicy);
 			Writer << "Namespace"sv << Namespace;
 
 			Writer.BeginArray("ChunkRequests"sv);
-			for (const GetCacheChunkRequest& ValueRequest : Requests)
+			if (OptionalValueFilter.empty())
 			{
-				Writer.BeginObject();
+				for (size_t ChunkIndex = 0; const GetCacheChunkRequest& ChunkRequest : Requests)
 				{
-					WriteCacheRequestKey(Writer, ValueRequest.Key);
+					Writer.BeginObject();
+					{
+						WriteCacheRequestKey(Writer, ChunkRequest.Key);
 
-					Writer.AddObjectId("ValueId"sv, ValueRequest.ValueId);
-					Writer.AddHash("ChunkId"sv, ValueRequest.ChunkId);
-					Writer.AddInteger("RawOffset"sv, ValueRequest.RawOffset);
-					Writer.AddInteger("RawSize"sv, ValueRequest.RawSize);
+						Writer.AddObjectId("ValueId"sv, ChunkRequest.ValueId);
+						Writer.AddHash("ChunkId"sv, ChunkRequest.ChunkId);
+						Writer.AddInteger("RawOffset"sv, ChunkRequest.RawOffset);
+						Writer.AddInteger("RawSize"sv, ChunkRequest.RawSize);
 
-					WriteCachePolicy(Writer, "Policy"sv, ValueRequest.Policy);
+						const std::optional<CachePolicy>& ChunkPolicy = Policy.ChunkPolicies[ChunkIndex++];
+
+						WriteCachePolicy(Writer, "Policy"sv, ChunkPolicy);
+					}
+					Writer.EndObject();
+				}
+			}
+			else
+			{
+				for (size_t Index : OptionalValueFilter)
+				{
+					const GetCacheChunkRequest& ChunkRequest = Requests[Index];
+					Writer.BeginObject();
+					{
+						WriteCacheRequestKey(Writer, ChunkRequest.Key);
+
+						Writer.AddObjectId("ValueId"sv, ChunkRequest.ValueId);
+						Writer.AddHash("ChunkId"sv, ChunkRequest.ChunkId);
+						Writer.AddInteger("RawOffset"sv, ChunkRequest.RawOffset);
+						Writer.AddInteger("RawSize"sv, ChunkRequest.RawSize);
+
+						const std::optional<CachePolicy>& ChunkPolicy = Policy.ChunkPolicies[Index];
+
+						WriteCachePolicy(Writer, "Policy"sv, ChunkPolicy);
+					}
+					Writer.EndObject();
 				}
-				Writer.EndObject();
 			}
 			Writer.EndArray();
 		}
@@ -1140,6 +1168,26 @@ namespace cacherequests {
 		return true;
 	}
 
+	static bool operator==(const ChunksRequestPolicy& Lhs, const ChunksRequestPolicy& Rhs)
+	{
+		if (Lhs.DefaultPolicy != Rhs.DefaultPolicy)
+		{
+			return false;
+		}
+		if (Lhs.ChunkPolicies.size() != Rhs.ChunkPolicies.size())
+		{
+			return false;
+		}
+		for (size_t Index = 0; Index < Lhs.ChunkPolicies.size(); ++Index)
+		{
+			if (Lhs.ChunkPolicies[Index] != Rhs.ChunkPolicies[Index])
+			{
+				return false;
+			}
+		}
+		return true;
+	}
+
 	static bool operator==(const PutCacheRecordsResult& Lhs, const PutCacheRecordsResult& Rhs) { return (Lhs.Success == Rhs.Success); }
 
 	static bool operator==(const GetCacheRecordsRequest& Lhs, const GetCacheRecordsRequest& Rhs)
@@ -1237,12 +1285,12 @@ namespace cacherequests {
 	static bool operator==(const GetCacheChunkRequest& Lhs, const GetCacheChunkRequest& Rhs)
 	{
 		return Lhs.Key == Rhs.Key && Lhs.ValueId == Rhs.ValueId && Lhs.ChunkId == Rhs.ChunkId && Lhs.RawOffset == Rhs.RawOffset &&
-			   Lhs.RawSize == Rhs.RawSize && Lhs.Policy == Rhs.Policy;
+			   Lhs.RawSize == Rhs.RawSize;
 	}
 
 	static bool operator==(const GetCacheChunksRequest& Lhs, const GetCacheChunksRequest& Rhs)
 	{
-		return Lhs.DefaultPolicy == Rhs.DefaultPolicy && Lhs.Namespace == Rhs.Namespace && Lhs.Requests == Rhs.Requests;
+		return Lhs.Namespace == Rhs.Namespace && Lhs.Requests == Rhs.Requests;
 	}
 
 	static CompressedBuffer MakeCompressedBuffer(size_t Size) { return CompressedBuffer::Compress(SharedBuffer(IoBuffer(Size))); };
@@ -1529,34 +1577,52 @@ namespace cacherequests {
 	TEST_CASE("cacherequests.get.cache.chunks")
 	{
 		GetCacheChunksRequest EmptyRequest;
+		ChunksRequestPolicy	  EmptyPolicy;
 		CbPackage			  EmptyRequestPackage;
-		CHECK(EmptyRequest.Format(EmptyRequestPackage));
+		CHECK(EmptyRequest.Format(EmptyRequestPackage, EmptyPolicy));
 		GetCacheChunksRequest EmptyRequestCopy;
-		CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage.GetObject()));  // Namespace is required
+		ChunksRequestPolicy	  EmptyPolicyCopy;
+		CHECK(!EmptyRequestCopy.Parse(EmptyRequestPackage.GetObject(), EmptyPolicyCopy));  // Namespace is required
+		CHECK(EmptyRequest == EmptyRequestCopy);
+		CHECK(EmptyPolicy == EmptyPolicyCopy);
 
 		GetCacheChunksRequest FullRequest = {
-			.DefaultPolicy = CachePolicy::StoreLocal,
-			.Namespace	   = "other_namespace",
-			.Requests	   = {{.Key		  = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")},
-							   .ValueId	  = Oid::NewOid(),
-							   .ChunkId	  = IoHash::FromHexString("ab3917854bfef7e7af2c372d795bb907a15cab15"),
-							   .RawOffset = 77,
-							   .RawSize	  = 33,
-							   .Policy	  = CachePolicy::Local},
+			.Namespace = "other_namespace",
+			.Requests  = {{.Key		  = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")},
+						   .ValueId	  = Oid::NewOid(),
+						   .ChunkId	  = IoHash::FromHexString("ab3917854bfef7e7af2c372d795bb907a15cab15"),
+						   .RawOffset = 77,
+						   .RawSize	  = 33},
 						  {.Key		= {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")},
-							   .ValueId = Oid::NewOid(),
-							   .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c"),
-							   .Policy	= CachePolicy::Remote},
+						   .ValueId = Oid::NewOid(),
+						   .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c")},
 						  {
-								  .Key	   = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")},
-								  .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c"),
+							  .Key	   = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")},
+							  .ChunkId = IoHash::FromHexString("372d795bb907a15cab15ab3917854bfef7e7af2c"),
 						  }}};
+		ChunksRequestPolicy FullPolicy = {.DefaultPolicy = CachePolicy::StoreLocal,
+										  .ChunkPolicies = {{CachePolicy::Local}, CachePolicy::Remote, {}}};
 
 		CbPackage FullRequestPackage;
-		CHECK(FullRequest.Format(FullRequestPackage));
+		CHECK(FullRequest.Format(FullRequestPackage, FullPolicy));
 		GetCacheChunksRequest FullRequestCopy;
-		CHECK(FullRequestCopy.Parse(FullRequestPackage.GetObject()));
+		ChunksRequestPolicy	  FullPolicyCopy;
+		CHECK(FullRequestCopy.Parse(FullRequestPackage.GetObject(), FullPolicyCopy));
 		CHECK(FullRequest == FullRequestCopy);
+		CHECK(FullPolicy == FullPolicyCopy);
+
+		CbPackage PartialRequestPackage;
+		CHECK(FullRequest.Format(PartialRequestPackage, FullPolicy, std::vector<size_t>({0, 2})));
+		GetCacheChunksRequest PartialRequestCopy;
+		ChunksRequestPolicy	  PartialPolicyCopy;
+		CHECK(PartialRequestCopy.Parse(PartialRequestPackage.GetObject(), PartialPolicyCopy));
+		CHECK(FullRequest.Namespace == PartialRequestCopy.Namespace);
+		CHECK(PartialRequestCopy.Requests.size() == 2);
+		CHECK(PartialRequestCopy.Requests[0] == FullRequestCopy.Requests[0]);
+		CHECK(PartialRequestCopy.Requests[1] == FullRequestCopy.Requests[2]);
+		CHECK(PartialPolicyCopy.ChunkPolicies.size() == 2);
+		CHECK(PartialPolicyCopy.ChunkPolicies[0] == FullPolicyCopy.ChunkPolicies[0]);
+		CHECK(PartialPolicyCopy.ChunkPolicies[1] == FullPolicyCopy.ChunkPolicies[2]);
 
 		GetCacheChunksResult EmptyResult;
 		CbPackage			 EmptyResponsePackage;
diff --git a/zenutil/include/zenutil/cache/cacherequests.h b/zenutil/include/zenutil/cache/cacherequests.h
index e11a0478b..ee153ed7d 100644
--- a/zenutil/include/zenutil/cache/cacherequests.h
+++ b/zenutil/include/zenutil/cache/cacherequests.h
@@ -246,25 +246,38 @@ namespace cacherequests {
 	//////////////////////////////////////////////////////////////////////////
 	// Get 1..n cache record values (attachments) for 1..n records
 
+	struct ChunksRequestPolicy
+	{
+		CachePolicy								DefaultPolicy = CachePolicy::Default;
+		std::vector<std::optional<CachePolicy>> ChunkPolicies;
+	};
+
+	inline CachePolicy GetEffectiveChunkPolicy(const ChunksRequestPolicy& Policy, size_t ChunkIndex)
+	{
+		if (!Policy.ChunkPolicies[ChunkIndex].has_value())
+		{
+			return Policy.DefaultPolicy;
+		}
+		return Policy.DefaultPolicy;
+	}
+
 	struct GetCacheChunkRequest
 	{
 		CacheKey Key;			  // Leave ValueId and ChunkId (RawHash) Zero to get the cache value
 		Oid ValueId = Oid::Zero;  // Set to get a record value if ChunkId is not known at request time, this requires a fetch of the cache
 								  // record to fulfill
-		IoHash					   ChunkId	 = IoHash::Zero;  // Set to get a record value directly without fetching cache record
-		uint64_t				   RawOffset = 0ull;
-		uint64_t				   RawSize	 = ~uint64_t(0);
-		std::optional<CachePolicy> Policy;
+		IoHash	 ChunkId   = IoHash::Zero;	// Set to get a record value directly without fetching cache record
+		uint64_t RawOffset = 0ull;
+		uint64_t RawSize   = ~uint64_t(0);
 	};
 
 	struct GetCacheChunksRequest
 	{
-		CachePolicy						  DefaultPolicy = CachePolicy::Default;
 		std::string						  Namespace;
 		std::vector<GetCacheChunkRequest> Requests;
 
-		bool Parse(const CbObjectView& BatchObject);
-		bool Format(CbPackage& OutPackage) const;
+		bool Parse(const CbObjectView& BatchObject, ChunksRequestPolicy& OutPolicy);
+		bool Format(CbPackage& OutPackage, const ChunksRequestPolicy& Policy, const std::span<const size_t> OptionalValueFilter = {}) const;
 	};
 
 	typedef CacheValuesResult GetCacheChunksResult;