Merge branch 'main' of https://github.com/EpicGames/zen

1 files changed, 280 insertions, 0 deletions

diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 17b8b76eb..7629e2ea1 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -1402,6 +1402,286 @@ TEST_CASE("zcache.cbpackage")
 	}
 }
 
+TEST_CASE("zcache.policy")
+{
+	using namespace std::literals;
+
+	struct ZenConfig
+	{
+		std::filesystem::path DataDir;
+		uint16_t			  Port;
+		std::string			  BaseUri;
+		std::string			  Args;
+
+		static ZenConfig New(uint16_t Port = 13337, std::string Args = "")
+		{
+			return ZenConfig{.DataDir = TestEnv.CreateNewTestDir(),
+							 .Port	  = Port,
+							 .BaseUri = "http://localhost:{}/z$"_format(Port),
+							 .Args	  = std::move(Args)};
+		}
+
+		static ZenConfig NewWithUpstream(uint16_t UpstreamPort)
+		{
+			return New(13337, "--upstream-thread-count=0 --upstream-zen-url=http://localhost:{}"_format(UpstreamPort));
+		}
+
+		void Spawn(ZenServerInstance& Inst)
+		{
+			Inst.SetTestDir(DataDir);
+			Inst.SpawnServer(Port, Args);
+			Inst.WaitUntilReady();
+		}
+	};
+
+	auto GenerateData = [](uint64_t Size, zen::IoHash& OutHash) -> zen::UniqueBuffer {
+		auto	 Buf  = zen::UniqueBuffer::Alloc(Size);
+		uint8_t* Data = reinterpret_cast<uint8_t*>(Buf.GetData());
+		for (uint64_t Idx = 0; Idx < Size; Idx++)
+		{
+			Data[Idx] = Idx % 256;
+		}
+		OutHash = zen::IoHash::HashBuffer(Data, Size);
+		return Buf;
+	};
+
+	auto GeneratePackage = [](zen::IoHash& OutAttachmentKey) -> zen::CbPackage {
+		auto Data			= zen::SharedBuffer::Clone(zen::MakeMemoryView<uint8_t>({1, 2, 3, 4, 5, 6, 7, 8, 9}));
+		auto CompressedData = zen::CompressedBuffer::Compress(Data);
+		OutAttachmentKey	= zen::IoHash::FromBLAKE3(CompressedData.GetRawHash());
+		zen::CbWriter Obj;
+		Obj.BeginObject("obj"sv);
+		Obj.AddBinaryAttachment("data", OutAttachmentKey);
+		Obj.EndObject();
+		zen::CbPackage Package;
+		Package.SetObject(Obj.Save().AsObject());
+		Package.AddAttachment(zen::CbAttachment(CompressedData));
+
+		return Package;
+	};
+
+	auto ToBuffer = [](zen::CbPackage Package) -> zen::IoBuffer {
+		zen::MemoryOutStream MemStream;
+		zen::BinaryWriter	 Writer(MemStream);
+		Package.Save(Writer);
+
+		return zen::IoBuffer(zen::IoBuffer::Clone, MemStream.Data(), MemStream.Size());
+	};
+
+	SUBCASE("query - 'local' does not query upstream (binary)")
+	{
+		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenServerInstance UpstreamInst(TestEnv);
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
+		ZenServerInstance LocalInst(TestEnv);
+		const auto		  Bucket = "legacy"sv;
+
+		UpstreamCfg.Spawn(UpstreamInst);
+		LocalCfg.Spawn(LocalInst);
+
+		zen::IoHash Key;
+		auto		BinaryValue = GenerateData(1024, Key);
+
+		// Store binary cache value upstream
+		{
+			cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(UpstreamCfg.BaseUri, Bucket, Key)},
+											cpr::Body{(const char*)BinaryValue.GetData(), BinaryValue.GetSize()},
+											cpr::Header{{"Content-Type", "application/octet-stream"}});
+			CHECK(Result.status_code == 201);
+		}
+
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?query=local"_format(LocalCfg.BaseUri, Bucket, Key)},
+											cpr::Header{{"Accept", "application/octet-stream"}});
+			CHECK(Result.status_code == 404);
+		}
+
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?query=local,remote"_format(LocalCfg.BaseUri, Bucket, Key)},
+											cpr::Header{{"Accept", "application/octet-stream"}});
+			CHECK(Result.status_code == 200);
+		}
+	}
+
+	SUBCASE("store - 'local' does not store upstream (binary)")
+	{
+		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenServerInstance UpstreamInst(TestEnv);
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
+		ZenServerInstance LocalInst(TestEnv);
+		const auto		  Bucket = "legacy"sv;
+
+		UpstreamCfg.Spawn(UpstreamInst);
+		LocalCfg.Spawn(LocalInst);
+
+		zen::IoHash Key;
+		auto		BinaryValue = GenerateData(1024, Key);
+
+		// Store binary cache value locally
+		{
+			cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}?store=local"_format(LocalCfg.BaseUri, Bucket, Key)},
+											cpr::Body{(const char*)BinaryValue.GetData(), BinaryValue.GetSize()},
+											cpr::Header{{"Content-Type", "application/octet-stream"}});
+			CHECK(Result.status_code == 201);
+		}
+
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}"_format(UpstreamCfg.BaseUri, Bucket, Key)},
+											cpr::Header{{"Accept", "application/octet-stream"}});
+			CHECK(Result.status_code == 404);
+		}
+
+		{
+			cpr::Response Result =
+				cpr::Get(cpr::Url{"{}/{}/{}"_format(LocalCfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/octet-stream"}});
+			CHECK(Result.status_code == 200);
+		}
+	}
+
+	SUBCASE("store - 'local/remote' stores local and upstream (binary)")
+	{
+		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenServerInstance UpstreamInst(TestEnv);
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
+		ZenServerInstance LocalInst(TestEnv);
+		const auto		  Bucket = "legacy"sv;
+
+		UpstreamCfg.Spawn(UpstreamInst);
+		LocalCfg.Spawn(LocalInst);
+
+		zen::IoHash Key;
+		auto		BinaryValue = GenerateData(1024, Key);
+
+		// Store binary cache value locally and upstream
+		{
+			cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}?store=local,remote"_format(LocalCfg.BaseUri, Bucket, Key)},
+											cpr::Body{(const char*)BinaryValue.GetData(), BinaryValue.GetSize()},
+											cpr::Header{{"Content-Type", "application/octet-stream"}});
+			CHECK(Result.status_code == 201);
+		}
+
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}"_format(UpstreamCfg.BaseUri, Bucket, Key)},
+											cpr::Header{{"Accept", "application/octet-stream"}});
+			CHECK(Result.status_code == 200);
+		}
+
+		{
+			cpr::Response Result =
+				cpr::Get(cpr::Url{"{}/{}/{}"_format(LocalCfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/octet-stream"}});
+			CHECK(Result.status_code == 200);
+		}
+	}
+
+	SUBCASE("query - 'local' does not query upstream (cppackage)")
+	{
+		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenServerInstance UpstreamInst(TestEnv);
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
+		ZenServerInstance LocalInst(TestEnv);
+		const auto		  Bucket = "legacy"sv;
+
+		UpstreamCfg.Spawn(UpstreamInst);
+		LocalCfg.Spawn(LocalInst);
+
+		zen::IoHash	   Key;
+		zen::CbPackage Package = GeneratePackage(Key);
+		auto		   Buf	   = ToBuffer(Package);
+
+		// Store package upstream
+		{
+			cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(UpstreamCfg.BaseUri, Bucket, Key)},
+											cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
+											cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 201);
+		}
+
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?query=local"_format(LocalCfg.BaseUri, Bucket, Key)},
+											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 404);
+		}
+
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?query=local,remote"_format(LocalCfg.BaseUri, Bucket, Key)},
+											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 200);
+		}
+	}
+
+	SUBCASE("store - 'local' does not store upstream (cbpackge)")
+	{
+		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenServerInstance UpstreamInst(TestEnv);
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
+		ZenServerInstance LocalInst(TestEnv);
+		const auto		  Bucket = "legacy"sv;
+
+		UpstreamCfg.Spawn(UpstreamInst);
+		LocalCfg.Spawn(LocalInst);
+
+		zen::IoHash	   Key;
+		zen::CbPackage Package = GeneratePackage(Key);
+		auto		   Buf	   = ToBuffer(Package);
+
+		// Store packge locally
+		{
+			cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}?store=local"_format(LocalCfg.BaseUri, Bucket, Key)},
+											cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
+											cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 201);
+		}
+
+		{
+			cpr::Response Result =
+				cpr::Get(cpr::Url{"{}/{}/{}"_format(UpstreamCfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 404);
+		}
+
+		{
+			cpr::Response Result =
+				cpr::Get(cpr::Url{"{}/{}/{}"_format(LocalCfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 200);
+		}
+	}
+
+	SUBCASE("store - 'local/remote' stores local and upstream (cbpackage)")
+	{
+		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenServerInstance UpstreamInst(TestEnv);
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
+		ZenServerInstance LocalInst(TestEnv);
+		const auto		  Bucket = "legacy"sv;
+
+		UpstreamCfg.Spawn(UpstreamInst);
+		LocalCfg.Spawn(LocalInst);
+
+		zen::IoHash	   Key;
+		zen::CbPackage Package = GeneratePackage(Key);
+		auto		   Buf	   = ToBuffer(Package);
+
+		// Store package locally and upstream
+		{
+			cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}?store=local,remote"_format(LocalCfg.BaseUri, Bucket, Key)},
+											cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
+											cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 201);
+		}
+
+		{
+			cpr::Response Result =
+				cpr::Get(cpr::Url{"{}/{}/{}"_format(UpstreamCfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 200);
+		}
+
+		{
+			cpr::Response Result =
+				cpr::Get(cpr::Url{"{}/{}/{}"_format(LocalCfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 200);
+		}
+	}
+}
+
 struct RemoteExecutionRequest
 {
 	RemoteExecutionRequest(std::string_view Host, int Port, std::filesystem::path& TreePath)