Merge branch 'main' of https://github.com/EpicGames/zen

2 files changed, 180 insertions, 171 deletions

diff --git a/zenserver-test/cachepolicy-tests.cpp b/zenserver-test/cachepolicy-tests.cpp
new file mode 100644
index 000000000..686ff818c
--- /dev/null
+++ b/zenserver-test/cachepolicy-tests.cpp
@@ -0,0 +1,164 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/zencore.h>
+
+#if ZEN_WITH_TESTS
+
+#	include <zencore/compactbinary.h>
+#	include <zencore/compactbinarybuilder.h>
+#	include <zencore/string.h>
+#	include <zencore/testing.h>
+#	include <zencore/uid.h>
+#	include <zenutil/cache/cachepolicy.h>
+
+namespace zen::tests {
+
+using namespace std::literals;
+
+TEST_CASE("cachepolicy")
+{
+	SUBCASE("atomics serialization")
+	{
+		CachePolicy SomeAtomics[] = {CachePolicy::None,
+									 CachePolicy::QueryLocal,
+									 CachePolicy::StoreRemote,
+									 CachePolicy::SkipData,
+									 CachePolicy::KeepAlive,
+									 CachePolicy::Disable};
+		for (CachePolicy Atomic : SomeAtomics)
+		{
+			CHECK(ParseCachePolicy(WriteToString<128>(Atomic)) == Atomic);
+		}
+		// Also verify that we ignore unrecognized bits
+		for (CachePolicy Atomic : SomeAtomics)
+		{
+			CHECK(ParseCachePolicy(WriteToString<128>(Atomic | (CachePolicy)0x10000000)) == Atomic);
+		}
+	}
+	SUBCASE("aliases serialization")
+	{
+		CachePolicy SomeAliases[] = {CachePolicy::Query, CachePolicy::Local};
+		for (CachePolicy Alias : SomeAliases)
+		{
+			CHECK(ParseCachePolicy(WriteToString<128>(Alias)) == Alias);
+		}
+		// Also verify that we ignore unrecognized bits
+		for (CachePolicy Alias : SomeAliases)
+		{
+			CHECK(ParseCachePolicy(WriteToString<128>(Alias | (CachePolicy)0x10000000)) == Alias);
+		}
+	}
+	SUBCASE("aliases take priority over atomics")
+	{
+		CHECK(WriteToString<128>(CachePolicy::Default).ToView() == "Default"sv);
+		CHECK(WriteToString<128>(CachePolicy::Query).ToView() == "Query"sv);
+		CHECK(WriteToString<128>(CachePolicy::Local).ToView() == "Local"sv);
+	}
+	SUBCASE("policies requiring multiple strings work")
+	{
+		char		Delimiter	= ',';
+		CachePolicy Combination = CachePolicy::SkipData | CachePolicy::QueryLocal;
+		CHECK(WriteToString<128>(Combination).ToView().find(Delimiter) != std::string_view::npos);
+		CHECK(ParseCachePolicy(WriteToString<128>(Combination)) == Combination);
+	}
+	SUBCASE("parsing invalid text")
+	{
+		CHECK(ParseCachePolicy(",,,") == CachePolicy::None);
+		CHECK(ParseCachePolicy("fee,fie,foo,fum") == CachePolicy::None);
+		CHECK(ParseCachePolicy("fee,KeepAlive,foo,fum") == CachePolicy::KeepAlive);
+	}
+}
+
+TEST_CASE("cacherecordpolicy")
+{
+	SUBCASE("policy with no values")
+	{
+		CachePolicy				 Policy = CachePolicy::SkipData | CachePolicy::QueryLocal;
+		CacheRecordPolicy		 RecordPolicy;
+		CacheRecordPolicyBuilder Builder(Policy);
+		RecordPolicy = Builder.Build();
+		SUBCASE("construct")
+		{
+			CHECK(RecordPolicy.IsUniform());
+			CHECK(RecordPolicy.GetRecordPolicy() == Policy);
+			CHECK(RecordPolicy.GetDefaultValuePolicy() == Policy);
+			CHECK(RecordPolicy.GetValuePolicy(Oid::NewOid()) == Policy);
+			CHECK(RecordPolicy.GetValuePolicies().size() == 0);
+		}
+		SUBCASE("saveload")
+		{
+			CbWriter Writer;
+			RecordPolicy.Save(Writer);
+			CbObject		  Saved	 = Writer.Save()->AsObject();
+			CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved);
+			CHECK(Loaded.IsUniform());
+			CHECK(Loaded.GetRecordPolicy() == Policy);
+			CHECK(Loaded.GetDefaultValuePolicy() == Policy);
+			CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == Policy);
+			CHECK(Loaded.GetValuePolicies().size() == 0);
+		}
+	}
+
+	SUBCASE("policy with values")
+	{
+		CachePolicy DefaultPolicy  = CachePolicy::StoreRemote | CachePolicy::QueryLocal;
+		CachePolicy PartialOverlap = CachePolicy::StoreRemote;
+		CachePolicy NoOverlap	   = CachePolicy::QueryRemote;
+		CachePolicy UnionPolicy	   = DefaultPolicy | PartialOverlap | NoOverlap;
+
+		CacheRecordPolicy		 RecordPolicy;
+		CacheRecordPolicyBuilder Builder(DefaultPolicy);
+		Oid						 PartialOid	  = Oid::NewOid();
+		Oid						 NoOverlapOid = Oid::NewOid();
+		Oid						 OtherOid	  = Oid::NewOid();
+		Builder.AddValuePolicy(PartialOid, PartialOverlap);
+		Builder.AddValuePolicy(NoOverlapOid, NoOverlap);
+		RecordPolicy = Builder.Build();
+		SUBCASE("construct")
+		{
+			CHECK(!RecordPolicy.IsUniform());
+			CHECK(RecordPolicy.GetRecordPolicy() == UnionPolicy);
+			CHECK(RecordPolicy.GetDefaultValuePolicy() == DefaultPolicy);
+			CHECK(RecordPolicy.GetValuePolicy(PartialOid) == PartialOverlap);
+			CHECK(RecordPolicy.GetValuePolicy(NoOverlapOid) == NoOverlap);
+			CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultPolicy);
+			CHECK(RecordPolicy.GetValuePolicies().size() == 2);
+		}
+		SUBCASE("saveload")
+		{
+			CbWriter Writer;
+			RecordPolicy.Save(Writer);
+			CbObject		  Saved	 = Writer.Save()->AsObject();
+			CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved);
+			CHECK(!RecordPolicy.IsUniform());
+			CHECK(RecordPolicy.GetRecordPolicy() == UnionPolicy);
+			CHECK(RecordPolicy.GetDefaultValuePolicy() == DefaultPolicy);
+			CHECK(RecordPolicy.GetValuePolicy(PartialOid) == PartialOverlap);
+			CHECK(RecordPolicy.GetValuePolicy(NoOverlapOid) == NoOverlap);
+			CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultPolicy);
+			CHECK(RecordPolicy.GetValuePolicies().size() == 2);
+		}
+	}
+
+	SUBCASE("parsing invalid text")
+	{
+		CacheRecordPolicy Loaded = CacheRecordPolicy::Load(CbObject());
+		CHECK(Loaded.IsUniform());
+		CHECK(Loaded.GetRecordPolicy() == CachePolicy::Default);
+		CHECK(Loaded.GetDefaultValuePolicy() == CachePolicy::Default);
+		CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == CachePolicy::Default);
+		CHECK(Loaded.GetValuePolicies().size() == 0);
+
+		CachePolicy Policy = CachePolicy::SkipData;
+		Loaded			   = CacheRecordPolicy::Load(CbObject(), Policy);
+		CHECK(Loaded.IsUniform());
+		CHECK(Loaded.GetRecordPolicy() == Policy);
+		CHECK(Loaded.GetDefaultValuePolicy() == Policy);
+		CHECK(Loaded.GetValuePolicy(Oid::NewOid()) == Policy);
+		CHECK(Loaded.GetValuePolicies().size() == 0);
+	}
+}
+
+}  // namespace zen::tests
+
+#endif
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 75aae6321..c1d48502e 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -1536,13 +1536,13 @@ TEST_CASE("zcache.policy")
 		}
 
 		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?query=local", LocalCfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=QueryLocal,Store", LocalCfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/octet-stream"}});
 			CHECK(Result.status_code == 404);
 		}
 
 		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?query=local,remote", LocalCfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,Store", LocalCfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/octet-stream"}});
 			CHECK(Result.status_code == 200);
 		}
@@ -1564,7 +1564,7 @@ TEST_CASE("zcache.policy")
 
 		// Store binary cache value locally
 		{
-			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?store=local", LocalCfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,StoreLocal", LocalCfg.BaseUri, Bucket, Key)},
 											cpr::Body{(const char*)BinaryValue.GetData(), BinaryValue.GetSize()},
 											cpr::Header{{"Content-Type", "application/octet-stream"}});
 			CHECK(Result.status_code == 201);
@@ -1599,7 +1599,7 @@ TEST_CASE("zcache.policy")
 
 		// Store binary cache value locally and upstream
 		{
-			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?store=local,remote", LocalCfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,Store", LocalCfg.BaseUri, Bucket, Key)},
 											cpr::Body{(const char*)BinaryValue.GetData(), BinaryValue.GetSize()},
 											cpr::Header{{"Content-Type", "application/octet-stream"}});
 			CHECK(Result.status_code == 201);
@@ -1643,13 +1643,13 @@ TEST_CASE("zcache.policy")
 		}
 
 		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?query=local", LocalCfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=QueryLocal,Store", LocalCfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
 			CHECK(Result.status_code == 404);
 		}
 
 		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?query=local,remote", LocalCfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,Store", LocalCfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
 			CHECK(Result.status_code == 200);
 		}
@@ -1673,7 +1673,7 @@ TEST_CASE("zcache.policy")
 
 		// Store packge locally
 		{
-			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?store=local", LocalCfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,StoreLocal", LocalCfg.BaseUri, Bucket, Key)},
 											cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
 											cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
 			CHECK(Result.status_code == 201);
@@ -1710,7 +1710,7 @@ TEST_CASE("zcache.policy")
 
 		// Store package locally and upstream
 		{
-			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?store=local,remote", LocalCfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}?Policy=Query,Store", LocalCfg.BaseUri, Bucket, Key)},
 											cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
 											cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
 			CHECK(Result.status_code == 201);
@@ -1729,129 +1729,6 @@ TEST_CASE("zcache.policy")
 		}
 	}
 
-	SUBCASE("skip - 'attachments' does not return attachments")
-	{
-		ZenConfig		  LocalCfg = ZenConfig::New();
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "texture"sv;
-
-		LocalCfg.Spawn(LocalInst);
-
-		zen::IoHash Key;
-		zen::IoHash PayloadId;
-
-		// Store package locally
-		{
-			zen::CbPackage Package = GeneratePackage(Key, PayloadId);
-			auto		   Buf	   = ToBuffer(Package);
-
-			CHECK(Package.GetAttachments().size() != 0);
-			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}", LocalCfg.BaseUri, Bucket, Key)},
-											cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
-											cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
-			CHECK(Result.status_code == 201);
-		}
-
-		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=attachments", LocalCfg.BaseUri, Bucket, Key)},
-											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
-			CHECK(Result.status_code == 200);
-
-			zen::IoBuffer  Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size());
-			zen::CbPackage Package;
-			const bool	   Ok = Package.TryLoad(Body);
-			CHECK(Ok);
-
-			CHECK(Ok);
-
-			CbObject			CacheRecord = Package.GetObject();
-			std::vector<IoHash> AttachmentKeys;
-
-			CacheRecord.IterateAttachments(
-				[&AttachmentKeys](CbFieldView AttachmentKey) { AttachmentKeys.push_back(AttachmentKey.AsHash()); });
-
-			CHECK(AttachmentKeys.size() != 0);
-			CHECK(Package.GetAttachments().size() == 0);
-		}
-
-		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}", LocalCfg.BaseUri, Bucket, Key)},
-											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
-			CHECK(Result.status_code == 200);
-
-			zen::IoBuffer  Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size());
-			zen::CbPackage Package;
-			const bool	   Ok = Package.TryLoad(Body);
-			CHECK(Ok);
-
-			CHECK(Ok);
-			CHECK(Package.GetAttachments().size() != 0);
-		}
-	}
-
-	SUBCASE("skip - 'attachments' does not return attachments when retrieved from upstream")
-	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
-		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "texture"sv;
-
-		UpstreamCfg.Spawn(UpstreamInst);
-		LocalCfg.Spawn(LocalInst);
-
-		zen::IoHash Key;
-		zen::IoHash PayloadId;
-
-		// Store package upstream
-		{
-			zen::CbPackage Package = GeneratePackage(Key, PayloadId);
-			auto		   Buf	   = ToBuffer(Package);
-
-			CHECK(Package.GetAttachments().size() != 0);
-			cpr::Response Result = cpr::Put(cpr::Url{fmt::format("{}/{}/{}", UpstreamCfg.BaseUri, Bucket, Key)},
-											cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
-											cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
-			CHECK(Result.status_code == 201);
-		}
-
-		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=attachments", LocalCfg.BaseUri, Bucket, Key)},
-											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
-			CHECK(Result.status_code == 200);
-
-			zen::IoBuffer  Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size());
-			zen::CbPackage Package;
-			const bool	   Ok = Package.TryLoad(Body);
-			CHECK(Ok);
-
-			CHECK(Ok);
-
-			CbObject			CacheRecord = Package.GetObject();
-			std::vector<IoHash> AttachmentKeys;
-
-			CacheRecord.IterateAttachments(
-				[&AttachmentKeys](CbFieldView AttachmentKey) { AttachmentKeys.push_back(AttachmentKey.AsHash()); });
-
-			CHECK(AttachmentKeys.size() != 0);
-			CHECK(Package.GetAttachments().size() == 0);
-		}
-
-		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}", LocalCfg.BaseUri, Bucket, Key)},
-											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
-			CHECK(Result.status_code == 200);
-
-			zen::IoBuffer  Body(zen::IoBuffer::Wrap, Result.text.data(), Result.text.size());
-			zen::CbPackage Package;
-			const bool	   Ok = Package.TryLoad(Body);
-			CHECK(Ok);
-
-			CHECK(Ok);
-			CHECK(Package.GetAttachments().size() != 0);
-		}
-	}
-
 	SUBCASE("skip - 'data' returns empty cache record/payload")
 	{
 		ZenConfig		  Cfg = ZenConfig::New();
@@ -1875,7 +1752,7 @@ TEST_CASE("zcache.policy")
 
 		// Get package
 		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=data", Cfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
 			CHECK(IsHttpSuccessCode(Result.status_code));
 			CHECK(Result.text.size() == 0);
@@ -1883,7 +1760,7 @@ TEST_CASE("zcache.policy")
 
 		// Get record
 		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=data", Cfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/x-ue-cbobject"}});
 			CHECK(IsHttpSuccessCode(Result.status_code));
 			CHECK(Result.text.size() == 0);
@@ -1891,8 +1768,9 @@ TEST_CASE("zcache.policy")
 
 		// Get payload
 		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}/{}?skip=data", Cfg.BaseUri, Bucket, Key, PayloadId)},
-											cpr::Header{{"Accept", "application/x-ue-comp"}});
+			cpr::Response Result =
+				cpr::Get(cpr::Url{fmt::format("{}/{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key, PayloadId)},
+						 cpr::Header{{"Accept", "application/x-ue-comp"}});
 			CHECK(IsHttpSuccessCode(Result.status_code));
 			CHECK(Result.text.size() == 0);
 		}
@@ -1919,7 +1797,7 @@ TEST_CASE("zcache.policy")
 
 		// Get package
 		{
-			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?skip=data", Cfg.BaseUri, Bucket, Key)},
+			cpr::Response Result = cpr::Get(cpr::Url{fmt::format("{}/{}/{}?Policy=Default,SkipData", Cfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/octet-stream"}});
 			CHECK(IsHttpSuccessCode(Result.status_code));
 			CHECK(Result.text.size() == 0);
@@ -2011,9 +1889,8 @@ TEST_CASE("zcache.rpc")
 		}
 		Request.EndArray();
 
-		Request.BeginObject("Policy");
-		CacheRecordPolicy::Save(Policy, Request);
-		Request.EndObject();
+		Request.SetName("Policy"sv);
+		Policy.Save(Request);
 
 		Request.EndObject();
 
@@ -2134,38 +2011,6 @@ TEST_CASE("zcache.rpc")
 		}
 	}
 
-	SUBCASE("policy - 'SkipAttachments' does not return any record attachments")
-	{
-		std::filesystem::path TestDir	 = TestEnv.CreateNewTestDir();
-		const uint16_t		  PortNumber = 13337;
-		const auto			  BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
-
-		ZenServerInstance Inst(TestEnv);
-		Inst.SetTestDir(TestDir);
-		Inst.SpawnServer(PortNumber);
-		Inst.WaitUntilReady();
-
-		CacheRecordPolicy		   Policy(CachePolicy::QueryLocal | CachePolicy::SkipAttachments);
-		std::vector<zen::CacheKey> Keys	  = PutCacheRecords(BaseUri, ""sv, "mastodon"sv, 4);
-		GetCacheRecordResult	   Result = GetCacheRecords(BaseUri, Keys, Policy);
-
-		CHECK(Result.Records.size() == Keys.size());
-
-		std::span<const zen::CbAttachment> Attachments = Result.Response.GetAttachments();
-		CHECK(Attachments.empty());
-
-		for (size_t Index = 0; CbFieldView RecordView : Result.Records)
-		{
-			const CacheKey& ExpectedKey = Keys[Index++];
-
-			CbObjectView   RecordObj = RecordView.AsObjectView();
-			CbObjectView   KeyObj	 = RecordObj["CacheKey"sv].AsObjectView();
-			const CacheKey Key		 = CacheKey::Create(KeyObj["Bucket"sv].AsString(), KeyObj["Hash"].AsHash());
-
-			CHECK(Key == ExpectedKey);
-		}
-	}
-
 	SUBCASE("policy - 'QueryLocal' does not query upstream")
 	{
 		using namespace utils;