Cache policy support (#47)

Add HandleRpc methods for the remaining ICacheStore requests from unreal: PutCacheValues/GetCacheValues. We now have batched versions for PutCacheRecords,GetCacheRecords,PutCacheValues,GetCacheValues,GetCacheChunks. Add support for CachePolicy flags to all of these batched methods.

* Add Batched PutCacheValues/GetCacheValues. Rename old GetCacheValues to GetCacheChunks.
* HandleRpcGetCacheRecords: Receive a CacheRecordPolicy with each key, and skipdata on attachments we already have.
* Changes to CachePolicy copied from Release-5.0 depot. Change serialization to use the key BasePolicy instead of DefaultValuePolicy.
* GetChunks: Read CacheRecords from remote if necessary to find ContentId. Implement QueryLocal, StoreLocal, and SkipData.

2 files changed, 76 insertions, 60 deletions

diff --git a/zenserver-test/cachepolicy-tests.cpp b/zenserver-test/cachepolicy-tests.cpp
index 686ff818c..d3135439c 100644
--- a/zenserver-test/cachepolicy-tests.cpp
+++ b/zenserver-test/cachepolicy-tests.cpp
@@ -23,8 +23,7 @@ TEST_CASE("cachepolicy")
 									 CachePolicy::QueryLocal,
 									 CachePolicy::StoreRemote,
 									 CachePolicy::SkipData,
-									 CachePolicy::KeepAlive,
-									 CachePolicy::Disable};
+									 CachePolicy::KeepAlive};
 		for (CachePolicy Atomic : SomeAtomics)
 		{
 			CHECK(ParseCachePolicy(WriteToString<128>(Atomic)) == Atomic);
@@ -73,7 +72,8 @@ TEST_CASE("cacherecordpolicy")
 {
 	SUBCASE("policy with no values")
 	{
-		CachePolicy				 Policy = CachePolicy::SkipData | CachePolicy::QueryLocal;
+		CachePolicy				 Policy = CachePolicy::SkipData | CachePolicy::QueryLocal | CachePolicy::PartialRecord;
+		CachePolicy				 ValuePolicy = Policy & CacheValuePolicy::PolicyMask;
 		CacheRecordPolicy		 RecordPolicy;
 		CacheRecordPolicyBuilder Builder(Policy);
 		RecordPolicy = Builder.Build();
@@ -81,8 +81,8 @@ TEST_CASE("cacherecordpolicy")
 		{
 			CHECK(RecordPolicy.IsUniform());
 			CHECK(RecordPolicy.GetRecordPolicy() == Policy);
-			CHECK(RecordPolicy.GetDefaultValuePolicy() == Policy);
-			CHECK(RecordPolicy.GetValuePolicy(Oid::NewOid()) == Policy);
+			CHECK(RecordPolicy.GetBasePolicy() == Policy);
+			CHECK(RecordPolicy.GetValuePolicy(Oid::NewOid()) == ValuePolicy);
 			CHECK(RecordPolicy.GetValuePolicies().size() == 0);
 		}
 		SUBCASE("saveload")
@@ -90,21 +90,22 @@ TEST_CASE("cacherecordpolicy")
 			CbWriter Writer;
 			RecordPolicy.Save(Writer);
 			CbObject		  Saved	 = Writer.Save()->AsObject();
-			CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved);
+			CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved).Get();
 			CHECK(Loaded.IsUniform());
 			CHECK(Loaded.GetRecordPolicy() == Policy);
-			CHECK(Loaded.GetDefaultValuePolicy() == Policy);
-			CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == Policy);
+			CHECK(Loaded.GetBasePolicy() == Policy);
+			CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == ValuePolicy);
 			CHECK(Loaded.GetValuePolicies().size() == 0);
 		}
 	}
 
 	SUBCASE("policy with values")
 	{
-		CachePolicy DefaultPolicy  = CachePolicy::StoreRemote | CachePolicy::QueryLocal;
+		CachePolicy DefaultPolicy  = CachePolicy::StoreRemote | CachePolicy::QueryLocal | CachePolicy::PartialRecord;
+		CachePolicy DefaultValuePolicy = DefaultPolicy & CacheValuePolicy::PolicyMask;
 		CachePolicy PartialOverlap = CachePolicy::StoreRemote;
 		CachePolicy NoOverlap	   = CachePolicy::QueryRemote;
-		CachePolicy UnionPolicy	   = DefaultPolicy | PartialOverlap | NoOverlap;
+		CachePolicy UnionPolicy	   = DefaultPolicy | PartialOverlap | NoOverlap | CachePolicy::PartialRecord;
 
 		CacheRecordPolicy		 RecordPolicy;
 		CacheRecordPolicyBuilder Builder(DefaultPolicy);
@@ -118,10 +119,10 @@ TEST_CASE("cacherecordpolicy")
 		{
 			CHECK(!RecordPolicy.IsUniform());
 			CHECK(RecordPolicy.GetRecordPolicy() == UnionPolicy);
-			CHECK(RecordPolicy.GetDefaultValuePolicy() == DefaultPolicy);
+			CHECK(RecordPolicy.GetBasePolicy() == DefaultPolicy);
 			CHECK(RecordPolicy.GetValuePolicy(PartialOid) == PartialOverlap);
 			CHECK(RecordPolicy.GetValuePolicy(NoOverlapOid) == NoOverlap);
-			CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultPolicy);
+			CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultValuePolicy);
 			CHECK(RecordPolicy.GetValuePolicies().size() == 2);
 		}
 		SUBCASE("saveload")
@@ -129,33 +130,21 @@ TEST_CASE("cacherecordpolicy")
 			CbWriter Writer;
 			RecordPolicy.Save(Writer);
 			CbObject		  Saved	 = Writer.Save()->AsObject();
-			CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved);
+			CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved).Get();
 			CHECK(!RecordPolicy.IsUniform());
 			CHECK(RecordPolicy.GetRecordPolicy() == UnionPolicy);
-			CHECK(RecordPolicy.GetDefaultValuePolicy() == DefaultPolicy);
+			CHECK(RecordPolicy.GetBasePolicy() == DefaultPolicy);
 			CHECK(RecordPolicy.GetValuePolicy(PartialOid) == PartialOverlap);
 			CHECK(RecordPolicy.GetValuePolicy(NoOverlapOid) == NoOverlap);
-			CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultPolicy);
+			CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultValuePolicy);
 			CHECK(RecordPolicy.GetValuePolicies().size() == 2);
 		}
 	}
 
 	SUBCASE("parsing invalid text")
 	{
-		CacheRecordPolicy Loaded = CacheRecordPolicy::Load(CbObject());
-		CHECK(Loaded.IsUniform());
-		CHECK(Loaded.GetRecordPolicy() == CachePolicy::Default);
-		CHECK(Loaded.GetDefaultValuePolicy() == CachePolicy::Default);
-		CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == CachePolicy::Default);
-		CHECK(Loaded.GetValuePolicies().size() == 0);
-
-		CachePolicy Policy = CachePolicy::SkipData;
-		Loaded			   = CacheRecordPolicy::Load(CbObject(), Policy);
-		CHECK(Loaded.IsUniform());
-		CHECK(Loaded.GetRecordPolicy() == Policy);
-		CHECK(Loaded.GetDefaultValuePolicy() == Policy);
-		CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == Policy);
-		CHECK(Loaded.GetValuePolicies().size() == 0);
+		OptionalCacheRecordPolicy Loaded = CacheRecordPolicy::Load(CbObject());
+		CHECK(Loaded.IsNull());
 	}
 }
 
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 85393aed2..425f43946 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -1827,7 +1827,8 @@ TEST_CASE("zcache.rpc")
 			Data[Idx] = Idx % 255;
 		}
 
-		zen::CbAttachment Attachment(zen::CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size())));
+		CompressedBuffer  Value = zen::CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size()));
+		zen::CbAttachment Attachment(Value);
 
 		Writer.BeginObject();
 		{
@@ -1838,7 +1839,17 @@ TEST_CASE("zcache.rpc")
 					Writer << "Bucket"sv << CacheKey.Bucket << "Hash"sv << CacheKey.Hash;
 				}
 				Writer.EndObject();
-				Writer << "Data"sv << Attachment;
+				Writer.BeginArray("Values"sv);
+				{
+					Writer.BeginObject();
+					{
+						Writer.AddObjectId("Id"sv, Oid::NewOid());
+						Writer.AddBinaryAttachment("RawHash"sv, IoHash::FromBLAKE3(Value.GetRawHash()));
+						Writer.AddInteger("RawSize"sv, Value.GetRawSize());
+					}
+					Writer.EndObject();
+				}
+				Writer.EndArray();
 			}
 			Writer.EndObject();
 			Writer.SetName("Policy"sv);
@@ -1862,7 +1873,7 @@ TEST_CASE("zcache.rpc")
 
 		for (uint32_t Key = 1; Key <= Num; ++Key)
 		{
-			zen::IoHash			KeyHash;
+			zen::IoHash KeyHash;
 			((uint32_t*)(KeyHash.Hash))[0] = Key;
 			const zen::CacheKey CacheKey   = zen::CacheKey::Create(Bucket, KeyHash);
 			CbPackage			Package;
@@ -1907,27 +1918,30 @@ TEST_CASE("zcache.rpc")
 		bool						  Success;
 	};
 
-	auto GetCacheRecords =
-		[](std::string_view BaseUri, std::span<zen::CacheKey> Keys, const zen::CacheRecordPolicy& Policy) -> GetCacheRecordResult {
+	auto GetCacheRecords = [](std::string_view BaseUri, std::span<zen::CacheKey> Keys, zen::CachePolicy Policy) -> GetCacheRecordResult {
 		using namespace zen;
 
 		CbObjectWriter Request;
 		Request << "Method"sv
 				<< "GetCacheRecords"sv;
 		Request.BeginObject("Params"sv);
-
-		Request.BeginArray("CacheKeys"sv);
-		for (const CacheKey& Key : Keys)
 		{
-			Request.BeginObject();
-			Request << "Bucket"sv << Key.Bucket << "Hash"sv << Key.Hash;
-			Request.EndObject();
+			Request << "DefaultPolicy"sv << WriteToString<128>(Policy);
+			Request.BeginArray("Requests"sv);
+			for (const CacheKey& Key : Keys)
+			{
+				Request.BeginObject();
+				{
+					Request.BeginObject("Key"sv);
+					{
+						Request << "Bucket"sv << Key.Bucket << "Hash"sv << Key.Hash;
+					}
+					Request.EndObject();
+				}
+				Request.EndObject();
+			}
+			Request.EndArray();
 		}
-		Request.EndArray();
-
-		Request.SetName("Policy"sv);
-		Policy.Save(Request);
-
 		Request.EndObject();
 
 		BinaryWriter Body;
@@ -1978,7 +1992,7 @@ TEST_CASE("zcache.rpc")
 		Inst.SpawnServer(PortNumber);
 		Inst.WaitUntilReady();
 
-		CacheRecordPolicy		   Policy;
+		CachePolicy				   Policy = CachePolicy::Default;
 		std::vector<zen::CacheKey> Keys	  = PutCacheRecords(BaseUri, "mastodon"sv, 128);
 		GetCacheRecordResult	   Result = GetCacheRecords(BaseUri, Keys, Policy);
 
@@ -1988,11 +2002,18 @@ TEST_CASE("zcache.rpc")
 		{
 			const CacheKey& ExpectedKey = Keys[Index++];
 
-			CbObjectView		RecordObj	   = RecordView.AsObjectView();
-			CbObjectView		KeyObj		   = RecordObj["Key"sv].AsObjectView();
-			const CacheKey		Key			   = CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"].AsHash());
-			const IoHash		AttachmentHash = RecordObj["Data"sv].AsHash();
-			const CbAttachment* Attachment	   = Result.Response.FindAttachment(AttachmentHash);
+			CbObjectView   RecordObj = RecordView.AsObjectView();
+			CbObjectView   KeyObj	 = RecordObj["Key"sv].AsObjectView();
+			const CacheKey Key		 = CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"].AsHash());
+			IoHash		   AttachmentHash;
+			size_t		   NumValues = 0;
+			for (CbFieldView Value : RecordObj["Values"sv])
+			{
+				AttachmentHash = Value.AsObjectView()["RawHash"sv].AsHash();
+				++NumValues;
+			}
+			CHECK(NumValues == 1);
+			const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash);
 
 			CHECK(Key == ExpectedKey);
 			CHECK(Attachment != nullptr);
@@ -2010,7 +2031,7 @@ TEST_CASE("zcache.rpc")
 		Inst.SpawnServer(PortNumber);
 		Inst.WaitUntilReady();
 
-		CacheRecordPolicy		   Policy;
+		CachePolicy				   Policy		= CachePolicy::Default;
 		std::vector<zen::CacheKey> ExistingKeys = PutCacheRecords(BaseUri, "mastodon"sv, 128);
 		std::vector<zen::CacheKey> Keys;
 
@@ -2035,12 +2056,18 @@ TEST_CASE("zcache.rpc")
 			}
 			else
 			{
-				const CacheKey&		ExpectedKey	   = ExistingKeys[KeyIndex++];
-				CbObjectView		RecordObj	   = RecordView.AsObjectView();
-				zen::CacheKey		Key			   = LoadKey(RecordObj["Key"sv]);
-				const IoHash		AttachmentHash = RecordObj["Data"sv].AsHash();
-				const CbAttachment* Attachment	   = Result.Response.FindAttachment(AttachmentHash);
-
+				const CacheKey& ExpectedKey = ExistingKeys[KeyIndex++];
+				CbObjectView	RecordObj	= RecordView.AsObjectView();
+				zen::CacheKey	Key			= LoadKey(RecordObj["Key"sv]);
+				IoHash			AttachmentHash;
+				size_t			NumValues = 0;
+				for (CbFieldView Value : RecordObj["Values"sv])
+				{
+					AttachmentHash = Value.AsObjectView()["RawHash"sv].AsHash();
+					++NumValues;
+				}
+				CHECK(NumValues == 1);
+				const CbAttachment* Attachment = Result.Response.FindAttachment(AttachmentHash);
 				CHECK(Key == ExpectedKey);
 				CHECK(Attachment != nullptr);
 			}
@@ -2061,7 +2088,7 @@ TEST_CASE("zcache.rpc")
 
 		std::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "mastodon"sv, 4);
 
-		CacheRecordPolicy	 Policy(CachePolicy::QueryLocal);
+		CachePolicy			 Policy = CachePolicy::QueryLocal;
 		GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, Keys, Policy);
 
 		CHECK(Result.Records.size() == Keys.size());
@@ -2086,7 +2113,7 @@ TEST_CASE("zcache.rpc")
 
 		std::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "mastodon"sv, 4);
 
-		CacheRecordPolicy	 Policy(CachePolicy::QueryLocal | CachePolicy::QueryRemote);
+		CachePolicy			 Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote);
 		GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, Keys, Policy);
 
 		CHECK(Result.Records.size() == Keys.size());