Merge pull request #899 from ue-foundation/zs/file-intern-extern-conversionHEADmain

Zs/file intern extern conversion

1 files changed, 407 insertions, 0 deletions

diff --git a/src/zenserver-test/projectstore-tests.cpp b/src/zenserver-test/projectstore-tests.cpp
index a37ecb6be..d72b897c8 100644
--- a/src/zenserver-test/projectstore-tests.cpp
+++ b/src/zenserver-test/projectstore-tests.cpp
@@ -22,6 +22,7 @@ ZEN_THIRD_PARTY_INCLUDES_START
 ZEN_THIRD_PARTY_INCLUDES_END
 
 #	include <random>
+#	include <thread>
 
 namespace zen::tests {
 
@@ -1154,6 +1155,412 @@ TEST_CASE("project.rpcappendop")
 	}
 }
 
+TEST_CASE("project.file.data.transitions")
+{
+	using namespace utils;
+
+	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestDir);
+	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady();
+
+	zen::StringBuilder<64> ServerBaseUri;
+	ServerBaseUri << fmt::format("http://localhost:{}", PortNumber);
+
+	// Set up a root directory with a test file on disk for path-referenced serving
+	std::filesystem::path RootDir		= TestDir / "root";
+	std::filesystem::path TestFilePath	= RootDir / "content" / "testfile.bin";
+	std::filesystem::path RelServerPath = std::filesystem::path("content") / "testfile.bin";
+	CreateDirectories(TestFilePath.parent_path());
+	IoBuffer FileBlob = CreateRandomBlob(4096);
+	WriteFile(TestFilePath, FileBlob);
+
+	// Create a compressed blob to use as a CAS-referenced attachment (content differs from FileBlob)
+	CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(2048)));
+
+	// Fixed chunk IDs for the file entry across sub-tests
+	const std::string_view FileChunkIdStr{
+		"aa000000"
+		"bb000000"
+		"cc000001"};
+	Oid FileOid = Oid::FromHexString(FileChunkIdStr);
+
+	HttpClient Http{ServerBaseUri};
+
+	auto MakeProject = [&](std::string_view ProjectName) {
+		CbObjectWriter Project;
+		Project.AddString("id"sv, ProjectName);
+		Project.AddString("root"sv, PathToUtf8(RootDir.c_str()));
+		Project.AddString("engine"sv, ""sv);
+		Project.AddString("project"sv, ""sv);
+		Project.AddString("projectfile"sv, ""sv);
+		HttpClient::Response Response = Http.Post(fmt::format("/prj/{}", ProjectName), Project.Save());
+		REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("MakeProject"));
+	};
+
+	auto MakeOplog = [&](std::string_view ProjectName, std::string_view OplogName) {
+		HttpClient::Response Response =
+			Http.Post(fmt::format("/prj/{}/oplog/{}", ProjectName, OplogName), IoBuffer{}, ZenContentType::kCbObject);
+		REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("MakeOplog"));
+	};
+
+	auto PostOplogEntry = [&](std::string_view ProjectName, std::string_view OplogName, const CbPackage& OpPackage) {
+		zen::BinaryWriter MemOut;
+		legacy::SaveCbPackage(OpPackage, MemOut);
+		IoBuffer Body{IoBuffer::Wrap, MemOut.GetData(), MemOut.GetSize()};
+		Body.SetContentType(HttpContentType::kCbPackage);
+		HttpClient::Response Response = Http.Post(fmt::format("/prj/{}/oplog/{}/new", ProjectName, OplogName), Body);
+		REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("PostOplogEntry"));
+	};
+
+	auto GetChunk = [&](std::string_view ProjectName) -> HttpClient::Response {
+		return Http.Get(fmt::format("/prj/{}/oplog/oplog/{}", ProjectName, FileChunkIdStr));
+	};
+
+	// Extract the raw decompressed bytes from a chunk response, handling both compressed and uncompressed payloads
+	auto GetDecompressedPayload = [](const HttpClient::Response& Response) -> IoBuffer {
+		if (Response.ResponsePayload.GetContentType() == ZenContentType::kCompressedBinary)
+		{
+			IoHash			 RawHash;
+			uint64_t		 RawSize;
+			CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Response.ResponsePayload), RawHash, RawSize);
+			REQUIRE(Compressed);
+			return Compressed.Decompress().AsIoBuffer();
+		}
+		return Response.ResponsePayload;
+	};
+
+	auto TriggerGcAndWait = [&]() {
+		HttpClient::Response TriggerResponse = Http.Post("/admin/gc?smallobjects=true"sv, IoBuffer{});
+		REQUIRE_MESSAGE(TriggerResponse.IsSuccess(), TriggerResponse.ErrorMessage("TriggerGc"));
+
+		for (int Attempt = 0; Attempt < 100; ++Attempt)
+		{
+			std::this_thread::sleep_for(std::chrono::milliseconds(100));
+			HttpClient::Response StatusResponse = Http.Get("/admin/gc"sv);
+			REQUIRE_MESSAGE(StatusResponse.IsSuccess(), StatusResponse.ErrorMessage("GcStatus"));
+			CbObject StatusObj = StatusResponse.AsObject();
+			if (StatusObj["Status"sv].AsString() == "Idle"sv)
+			{
+				return;
+			}
+		}
+		FAIL("GC did not complete within timeout");
+	};
+
+	auto BuildPathReferencedFileOp = [&](const Oid& KeyId) -> CbPackage {
+		CbPackage	   Package;
+		CbObjectWriter Object;
+		Object << "key"sv << OidAsString(KeyId);
+		Object.BeginArray("files"sv);
+		Object.BeginObject();
+		Object << "id"sv << FileOid;
+		Object << "serverpath"sv << RelServerPath.string();
+		Object << "clientpath"sv
+			   << "/{engine}/testfile.bin"sv;
+		Object.EndObject();
+		Object.EndArray();
+		Package.SetObject(Object.Save());
+		return Package;
+	};
+
+	auto BuildHashReferencedFileOp = [&](const Oid& KeyId, const CompressedBuffer& Blob) -> CbPackage {
+		CbPackage	   Package;
+		CbObjectWriter Object;
+		Object << "key"sv << OidAsString(KeyId);
+		CbAttachment Attach(Blob, Blob.DecodeRawHash());
+		Object.BeginArray("files"sv);
+		Object.BeginObject();
+		Object << "id"sv << FileOid;
+		Object << "data"sv << Attach;
+		Object << "clientpath"sv
+			   << "/{engine}/testfile.bin"sv;
+		Object.EndObject();
+		Object.EndArray();
+		Package.AddAttachment(Attach);
+		Package.SetObject(Object.Save());
+		return Package;
+	};
+
+	SUBCASE("path-referenced file is retrievable")
+	{
+		MakeProject("proj_path"sv);
+		MakeOplog("proj_path"sv, "oplog"sv);
+
+		CbPackage Op = BuildPathReferencedFileOp(Oid::NewOid());
+		PostOplogEntry("proj_path"sv, "oplog"sv, Op);
+
+		HttpClient::Response Response = GetChunk("proj_path"sv);
+		CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk"));
+		if (Response.IsSuccess())
+		{
+			IoBuffer Payload = GetDecompressedPayload(Response);
+			CHECK_EQ(Payload.GetSize(), FileBlob.GetSize());
+			CHECK(Payload.GetView().EqualBytes(FileBlob.GetView()));
+		}
+	}
+
+	SUBCASE("hash-referenced file is retrievable")
+	{
+		MakeProject("proj_hash"sv);
+		MakeOplog("proj_hash"sv, "oplog"sv);
+
+		CbPackage Op = BuildHashReferencedFileOp(Oid::NewOid(), CompressedBlob);
+		PostOplogEntry("proj_hash"sv, "oplog"sv, Op);
+
+		HttpClient::Response Response = GetChunk("proj_hash"sv);
+		CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk"));
+		if (Response.IsSuccess())
+		{
+			IoBuffer Payload			  = GetDecompressedPayload(Response);
+			IoBuffer ExpectedDecompressed = CompressedBlob.Decompress().AsIoBuffer();
+			CHECK_EQ(Payload.GetSize(), ExpectedDecompressed.GetSize());
+			CHECK(Payload.GetView().EqualBytes(ExpectedDecompressed.GetView()));
+		}
+	}
+
+	SUBCASE("hash-referenced to path-referenced transition with different content")
+	{
+		MakeProject("proj_hash_to_path_diff"sv);
+		MakeOplog("proj_hash_to_path_diff"sv, "oplog"sv);
+
+		Oid	 FirstOpKey = Oid::NewOid();
+		Oid	 SecondOpKey;
+		bool RunGcAfterTransition = false;
+
+		SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); }
+		SUBCASE("same op key") { SecondOpKey = FirstOpKey; }
+		SUBCASE("new op key with gc")
+		{
+			SecondOpKey			 = Oid::NewOid();
+			RunGcAfterTransition = true;
+		}
+		SUBCASE("same op key with gc")
+		{
+			SecondOpKey			 = FirstOpKey;
+			RunGcAfterTransition = true;
+		}
+
+		// First op: file with CAS hash (content differs from the on-disk file)
+		{
+			CbPackage Op = BuildHashReferencedFileOp(FirstOpKey, CompressedBlob);
+			PostOplogEntry("proj_hash_to_path_diff"sv, "oplog"sv, Op);
+
+			HttpClient::Response Response = GetChunk("proj_hash_to_path_diff"sv);
+			CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op"));
+			if (Response.IsSuccess())
+			{
+				IoBuffer Payload			  = GetDecompressedPayload(Response);
+				IoBuffer ExpectedDecompressed = CompressedBlob.Decompress().AsIoBuffer();
+				CHECK(Payload.GetView().EqualBytes(ExpectedDecompressed.GetView()));
+			}
+		}
+
+		// Second op: same FileId transitions to serverpath (different data)
+		{
+			CbPackage Op = BuildPathReferencedFileOp(SecondOpKey);
+			PostOplogEntry("proj_hash_to_path_diff"sv, "oplog"sv, Op);
+		}
+
+		if (RunGcAfterTransition)
+		{
+			TriggerGcAndWait();
+		}
+
+		// Must serve the on-disk file content, not the old CAS blob
+		HttpClient::Response Response = GetChunk("proj_hash_to_path_diff"sv);
+		CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition"));
+		if (Response.IsSuccess())
+		{
+			IoBuffer Payload = GetDecompressedPayload(Response);
+			CHECK_EQ(Payload.GetSize(), FileBlob.GetSize());
+			CHECK(Payload.GetView().EqualBytes(FileBlob.GetView()));
+		}
+	}
+
+	SUBCASE("hash-referenced to path-referenced transition with identical content")
+	{
+		// Compress the same on-disk file content as a CAS blob so both references yield identical data
+		CompressedBuffer MatchingBlob = CompressedBuffer::Compress(SharedBuffer::Clone(FileBlob.GetView()));
+
+		MakeProject("proj_hash_to_path_same"sv);
+		MakeOplog("proj_hash_to_path_same"sv, "oplog"sv);
+
+		Oid	 FirstOpKey = Oid::NewOid();
+		Oid	 SecondOpKey;
+		bool RunGcAfterTransition = false;
+
+		SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); }
+		SUBCASE("same op key") { SecondOpKey = FirstOpKey; }
+		SUBCASE("new op key with gc")
+		{
+			SecondOpKey			 = Oid::NewOid();
+			RunGcAfterTransition = true;
+		}
+		SUBCASE("same op key with gc")
+		{
+			SecondOpKey			 = FirstOpKey;
+			RunGcAfterTransition = true;
+		}
+
+		// First op: file with CAS hash (content matches the on-disk file)
+		{
+			CbPackage Op = BuildHashReferencedFileOp(FirstOpKey, MatchingBlob);
+			PostOplogEntry("proj_hash_to_path_same"sv, "oplog"sv, Op);
+
+			HttpClient::Response Response = GetChunk("proj_hash_to_path_same"sv);
+			CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op"));
+			if (Response.IsSuccess())
+			{
+				IoBuffer Payload = GetDecompressedPayload(Response);
+				CHECK(Payload.GetView().EqualBytes(FileBlob.GetView()));
+			}
+		}
+
+		// Second op: same FileId transitions to serverpath (same data)
+		{
+			CbPackage Op = BuildPathReferencedFileOp(SecondOpKey);
+			PostOplogEntry("proj_hash_to_path_same"sv, "oplog"sv, Op);
+		}
+
+		if (RunGcAfterTransition)
+		{
+			TriggerGcAndWait();
+		}
+
+		// Must still resolve successfully after the transition
+		HttpClient::Response Response = GetChunk("proj_hash_to_path_same"sv);
+		CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition"));
+		if (Response.IsSuccess())
+		{
+			IoBuffer Payload = GetDecompressedPayload(Response);
+			CHECK_EQ(Payload.GetSize(), FileBlob.GetSize());
+			CHECK(Payload.GetView().EqualBytes(FileBlob.GetView()));
+		}
+	}
+
+	SUBCASE("path-referenced to hash-referenced transition with different content")
+	{
+		MakeProject("proj_path_to_hash_diff"sv);
+		MakeOplog("proj_path_to_hash_diff"sv, "oplog"sv);
+
+		Oid	 FirstOpKey = Oid::NewOid();
+		Oid	 SecondOpKey;
+		bool RunGcAfterTransition = false;
+
+		SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); }
+		SUBCASE("same op key") { SecondOpKey = FirstOpKey; }
+		SUBCASE("new op key with gc")
+		{
+			SecondOpKey			 = Oid::NewOid();
+			RunGcAfterTransition = true;
+		}
+		SUBCASE("same op key with gc")
+		{
+			SecondOpKey			 = FirstOpKey;
+			RunGcAfterTransition = true;
+		}
+
+		// First op: file with serverpath
+		{
+			CbPackage Op = BuildPathReferencedFileOp(FirstOpKey);
+			PostOplogEntry("proj_path_to_hash_diff"sv, "oplog"sv, Op);
+
+			HttpClient::Response Response = GetChunk("proj_path_to_hash_diff"sv);
+			CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op"));
+			if (Response.IsSuccess())
+			{
+				IoBuffer Payload = GetDecompressedPayload(Response);
+				CHECK(Payload.GetView().EqualBytes(FileBlob.GetView()));
+			}
+		}
+
+		// Second op: same FileId transitions to CAS hash (different data)
+		{
+			CbPackage Op = BuildHashReferencedFileOp(SecondOpKey, CompressedBlob);
+			PostOplogEntry("proj_path_to_hash_diff"sv, "oplog"sv, Op);
+		}
+
+		if (RunGcAfterTransition)
+		{
+			TriggerGcAndWait();
+		}
+
+		// Must serve the CAS blob content, not the old on-disk file
+		HttpClient::Response Response = GetChunk("proj_path_to_hash_diff"sv);
+		CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition"));
+		if (Response.IsSuccess())
+		{
+			IoBuffer Payload			  = GetDecompressedPayload(Response);
+			IoBuffer ExpectedDecompressed = CompressedBlob.Decompress().AsIoBuffer();
+			CHECK_EQ(Payload.GetSize(), ExpectedDecompressed.GetSize());
+			CHECK(Payload.GetView().EqualBytes(ExpectedDecompressed.GetView()));
+		}
+	}
+
+	SUBCASE("path-referenced to hash-referenced transition with identical content")
+	{
+		// Compress the same on-disk file content as a CAS blob so both references yield identical data
+		CompressedBuffer MatchingBlob = CompressedBuffer::Compress(SharedBuffer::Clone(FileBlob.GetView()));
+
+		MakeProject("proj_path_to_hash_same"sv);
+		MakeOplog("proj_path_to_hash_same"sv, "oplog"sv);
+
+		Oid	 FirstOpKey = Oid::NewOid();
+		Oid	 SecondOpKey;
+		bool RunGcAfterTransition = false;
+
+		SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); }
+		SUBCASE("same op key") { SecondOpKey = FirstOpKey; }
+		SUBCASE("new op key with gc")
+		{
+			SecondOpKey			 = Oid::NewOid();
+			RunGcAfterTransition = true;
+		}
+		SUBCASE("same op key with gc")
+		{
+			SecondOpKey			 = FirstOpKey;
+			RunGcAfterTransition = true;
+		}
+
+		// First op: file with serverpath
+		{
+			CbPackage Op = BuildPathReferencedFileOp(FirstOpKey);
+			PostOplogEntry("proj_path_to_hash_same"sv, "oplog"sv, Op);
+
+			HttpClient::Response Response = GetChunk("proj_path_to_hash_same"sv);
+			CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op"));
+			if (Response.IsSuccess())
+			{
+				IoBuffer Payload = GetDecompressedPayload(Response);
+				CHECK(Payload.GetView().EqualBytes(FileBlob.GetView()));
+			}
+		}
+
+		// Second op: same FileId transitions to CAS hash (same data)
+		{
+			CbPackage Op = BuildHashReferencedFileOp(SecondOpKey, MatchingBlob);
+			PostOplogEntry("proj_path_to_hash_same"sv, "oplog"sv, Op);
+		}
+
+		if (RunGcAfterTransition)
+		{
+			TriggerGcAndWait();
+		}
+
+		// Must still resolve successfully after the transition
+		HttpClient::Response Response = GetChunk("proj_path_to_hash_same"sv);
+		CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition"));
+		if (Response.IsSuccess())
+		{
+			IoBuffer Payload = GetDecompressedPayload(Response);
+			CHECK_EQ(Payload.GetSize(), FileBlob.GetSize());
+			CHECK(Payload.GetView().EqualBytes(FileBlob.GetView()));
+		}
+	}
+}
+
 TEST_SUITE_END();
 
 }  // namespace zen::tests