From c69187555ca580b560e220c4d4e4265821021650 Mon Sep 17 00:00:00 2001 From: zousar Date: Sat, 28 Mar 2026 01:35:21 -0600 Subject: Test cases for transitioning file reference types --- src/zenserver-test/projectstore-tests.cpp | 407 ++++++++++++++++++++++++++++++ 1 file changed, 407 insertions(+) (limited to 'src/zenserver-test/projectstore-tests.cpp') diff --git a/src/zenserver-test/projectstore-tests.cpp b/src/zenserver-test/projectstore-tests.cpp index a37ecb6be..d72b897c8 100644 --- a/src/zenserver-test/projectstore-tests.cpp +++ b/src/zenserver-test/projectstore-tests.cpp @@ -22,6 +22,7 @@ ZEN_THIRD_PARTY_INCLUDES_START ZEN_THIRD_PARTY_INCLUDES_END # include +# include namespace zen::tests { @@ -1154,6 +1155,412 @@ TEST_CASE("project.rpcappendop") } } +TEST_CASE("project.file.data.transitions") +{ + using namespace utils; + + std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(); + + zen::StringBuilder<64> ServerBaseUri; + ServerBaseUri << fmt::format("http://localhost:{}", PortNumber); + + // Set up a root directory with a test file on disk for path-referenced serving + std::filesystem::path RootDir = TestDir / "root"; + std::filesystem::path TestFilePath = RootDir / "content" / "testfile.bin"; + std::filesystem::path RelServerPath = std::filesystem::path("content") / "testfile.bin"; + CreateDirectories(TestFilePath.parent_path()); + IoBuffer FileBlob = CreateRandomBlob(4096); + WriteFile(TestFilePath, FileBlob); + + // Create a compressed blob to use as a CAS-referenced attachment (content differs from FileBlob) + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(2048))); + + // Fixed chunk IDs for the file entry across sub-tests + const std::string_view FileChunkIdStr{ + "aa000000" + "bb000000" + "cc000001"}; + Oid FileOid = Oid::FromHexString(FileChunkIdStr); + + HttpClient Http{ServerBaseUri}; + + auto MakeProject = [&](std::string_view ProjectName) { + CbObjectWriter Project; + Project.AddString("id"sv, ProjectName); + Project.AddString("root"sv, PathToUtf8(RootDir.c_str())); + Project.AddString("engine"sv, ""sv); + Project.AddString("project"sv, ""sv); + Project.AddString("projectfile"sv, ""sv); + HttpClient::Response Response = Http.Post(fmt::format("/prj/{}", ProjectName), Project.Save()); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("MakeProject")); + }; + + auto MakeOplog = [&](std::string_view ProjectName, std::string_view OplogName) { + HttpClient::Response Response = + Http.Post(fmt::format("/prj/{}/oplog/{}", ProjectName, OplogName), IoBuffer{}, ZenContentType::kCbObject); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("MakeOplog")); + }; + + auto PostOplogEntry = [&](std::string_view ProjectName, std::string_view OplogName, const CbPackage& OpPackage) { + zen::BinaryWriter MemOut; + legacy::SaveCbPackage(OpPackage, MemOut); + IoBuffer Body{IoBuffer::Wrap, MemOut.GetData(), MemOut.GetSize()}; + Body.SetContentType(HttpContentType::kCbPackage); + HttpClient::Response Response = Http.Post(fmt::format("/prj/{}/oplog/{}/new", ProjectName, OplogName), Body); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("PostOplogEntry")); + }; + + auto GetChunk = [&](std::string_view ProjectName) -> HttpClient::Response { + return Http.Get(fmt::format("/prj/{}/oplog/oplog/{}", ProjectName, FileChunkIdStr)); + }; + + // Extract the raw decompressed bytes from a chunk response, handling both compressed and uncompressed payloads + auto GetDecompressedPayload = [](const HttpClient::Response& Response) -> IoBuffer { + if (Response.ResponsePayload.GetContentType() == ZenContentType::kCompressedBinary) + { + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Response.ResponsePayload), RawHash, RawSize); + REQUIRE(Compressed); + return Compressed.Decompress().AsIoBuffer(); + } + return Response.ResponsePayload; + }; + + auto TriggerGcAndWait = [&]() { + HttpClient::Response TriggerResponse = Http.Post("/admin/gc?smallobjects=true"sv, IoBuffer{}); + REQUIRE_MESSAGE(TriggerResponse.IsSuccess(), TriggerResponse.ErrorMessage("TriggerGc")); + + for (int Attempt = 0; Attempt < 100; ++Attempt) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + HttpClient::Response StatusResponse = Http.Get("/admin/gc"sv); + REQUIRE_MESSAGE(StatusResponse.IsSuccess(), StatusResponse.ErrorMessage("GcStatus")); + CbObject StatusObj = StatusResponse.AsObject(); + if (StatusObj["Status"sv].AsString() == "Idle"sv) + { + return; + } + } + FAIL("GC did not complete within timeout"); + }; + + auto BuildPathReferencedFileOp = [&](const Oid& KeyId) -> CbPackage { + CbPackage Package; + CbObjectWriter Object; + Object << "key"sv << OidAsString(KeyId); + Object.BeginArray("files"sv); + Object.BeginObject(); + Object << "id"sv << FileOid; + Object << "serverpath"sv << RelServerPath.string(); + Object << "clientpath"sv + << "/{engine}/testfile.bin"sv; + Object.EndObject(); + Object.EndArray(); + Package.SetObject(Object.Save()); + return Package; + }; + + auto BuildHashReferencedFileOp = [&](const Oid& KeyId, const CompressedBuffer& Blob) -> CbPackage { + CbPackage Package; + CbObjectWriter Object; + Object << "key"sv << OidAsString(KeyId); + CbAttachment Attach(Blob, Blob.DecodeRawHash()); + Object.BeginArray("files"sv); + Object.BeginObject(); + Object << "id"sv << FileOid; + Object << "data"sv << Attach; + Object << "clientpath"sv + << "/{engine}/testfile.bin"sv; + Object.EndObject(); + Object.EndArray(); + Package.AddAttachment(Attach); + Package.SetObject(Object.Save()); + return Package; + }; + + SUBCASE("path-referenced file is retrievable") + { + MakeProject("proj_path"sv); + MakeOplog("proj_path"sv, "oplog"sv); + + CbPackage Op = BuildPathReferencedFileOp(Oid::NewOid()); + PostOplogEntry("proj_path"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_path"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK_EQ(Payload.GetSize(), FileBlob.GetSize()); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + SUBCASE("hash-referenced file is retrievable") + { + MakeProject("proj_hash"sv); + MakeOplog("proj_hash"sv, "oplog"sv); + + CbPackage Op = BuildHashReferencedFileOp(Oid::NewOid(), CompressedBlob); + PostOplogEntry("proj_hash"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_hash"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + IoBuffer ExpectedDecompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK_EQ(Payload.GetSize(), ExpectedDecompressed.GetSize()); + CHECK(Payload.GetView().EqualBytes(ExpectedDecompressed.GetView())); + } + } + + SUBCASE("hash-referenced to path-referenced transition with different content") + { + MakeProject("proj_hash_to_path_diff"sv); + MakeOplog("proj_hash_to_path_diff"sv, "oplog"sv); + + Oid FirstOpKey = Oid::NewOid(); + Oid SecondOpKey; + bool RunGcAfterTransition = false; + + SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); } + SUBCASE("same op key") { SecondOpKey = FirstOpKey; } + SUBCASE("new op key with gc") + { + SecondOpKey = Oid::NewOid(); + RunGcAfterTransition = true; + } + SUBCASE("same op key with gc") + { + SecondOpKey = FirstOpKey; + RunGcAfterTransition = true; + } + + // First op: file with CAS hash (content differs from the on-disk file) + { + CbPackage Op = BuildHashReferencedFileOp(FirstOpKey, CompressedBlob); + PostOplogEntry("proj_hash_to_path_diff"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_hash_to_path_diff"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + IoBuffer ExpectedDecompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(Payload.GetView().EqualBytes(ExpectedDecompressed.GetView())); + } + } + + // Second op: same FileId transitions to serverpath (different data) + { + CbPackage Op = BuildPathReferencedFileOp(SecondOpKey); + PostOplogEntry("proj_hash_to_path_diff"sv, "oplog"sv, Op); + } + + if (RunGcAfterTransition) + { + TriggerGcAndWait(); + } + + // Must serve the on-disk file content, not the old CAS blob + HttpClient::Response Response = GetChunk("proj_hash_to_path_diff"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK_EQ(Payload.GetSize(), FileBlob.GetSize()); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + SUBCASE("hash-referenced to path-referenced transition with identical content") + { + // Compress the same on-disk file content as a CAS blob so both references yield identical data + CompressedBuffer MatchingBlob = CompressedBuffer::Compress(SharedBuffer::Clone(FileBlob.GetView())); + + MakeProject("proj_hash_to_path_same"sv); + MakeOplog("proj_hash_to_path_same"sv, "oplog"sv); + + Oid FirstOpKey = Oid::NewOid(); + Oid SecondOpKey; + bool RunGcAfterTransition = false; + + SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); } + SUBCASE("same op key") { SecondOpKey = FirstOpKey; } + SUBCASE("new op key with gc") + { + SecondOpKey = Oid::NewOid(); + RunGcAfterTransition = true; + } + SUBCASE("same op key with gc") + { + SecondOpKey = FirstOpKey; + RunGcAfterTransition = true; + } + + // First op: file with CAS hash (content matches the on-disk file) + { + CbPackage Op = BuildHashReferencedFileOp(FirstOpKey, MatchingBlob); + PostOplogEntry("proj_hash_to_path_same"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_hash_to_path_same"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + // Second op: same FileId transitions to serverpath (same data) + { + CbPackage Op = BuildPathReferencedFileOp(SecondOpKey); + PostOplogEntry("proj_hash_to_path_same"sv, "oplog"sv, Op); + } + + if (RunGcAfterTransition) + { + TriggerGcAndWait(); + } + + // Must still resolve successfully after the transition + HttpClient::Response Response = GetChunk("proj_hash_to_path_same"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK_EQ(Payload.GetSize(), FileBlob.GetSize()); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + SUBCASE("path-referenced to hash-referenced transition with different content") + { + MakeProject("proj_path_to_hash_diff"sv); + MakeOplog("proj_path_to_hash_diff"sv, "oplog"sv); + + Oid FirstOpKey = Oid::NewOid(); + Oid SecondOpKey; + bool RunGcAfterTransition = false; + + SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); } + SUBCASE("same op key") { SecondOpKey = FirstOpKey; } + SUBCASE("new op key with gc") + { + SecondOpKey = Oid::NewOid(); + RunGcAfterTransition = true; + } + SUBCASE("same op key with gc") + { + SecondOpKey = FirstOpKey; + RunGcAfterTransition = true; + } + + // First op: file with serverpath + { + CbPackage Op = BuildPathReferencedFileOp(FirstOpKey); + PostOplogEntry("proj_path_to_hash_diff"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_path_to_hash_diff"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + // Second op: same FileId transitions to CAS hash (different data) + { + CbPackage Op = BuildHashReferencedFileOp(SecondOpKey, CompressedBlob); + PostOplogEntry("proj_path_to_hash_diff"sv, "oplog"sv, Op); + } + + if (RunGcAfterTransition) + { + TriggerGcAndWait(); + } + + // Must serve the CAS blob content, not the old on-disk file + HttpClient::Response Response = GetChunk("proj_path_to_hash_diff"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + IoBuffer ExpectedDecompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK_EQ(Payload.GetSize(), ExpectedDecompressed.GetSize()); + CHECK(Payload.GetView().EqualBytes(ExpectedDecompressed.GetView())); + } + } + + SUBCASE("path-referenced to hash-referenced transition with identical content") + { + // Compress the same on-disk file content as a CAS blob so both references yield identical data + CompressedBuffer MatchingBlob = CompressedBuffer::Compress(SharedBuffer::Clone(FileBlob.GetView())); + + MakeProject("proj_path_to_hash_same"sv); + MakeOplog("proj_path_to_hash_same"sv, "oplog"sv); + + Oid FirstOpKey = Oid::NewOid(); + Oid SecondOpKey; + bool RunGcAfterTransition = false; + + SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); } + SUBCASE("same op key") { SecondOpKey = FirstOpKey; } + SUBCASE("new op key with gc") + { + SecondOpKey = Oid::NewOid(); + RunGcAfterTransition = true; + } + SUBCASE("same op key with gc") + { + SecondOpKey = FirstOpKey; + RunGcAfterTransition = true; + } + + // First op: file with serverpath + { + CbPackage Op = BuildPathReferencedFileOp(FirstOpKey); + PostOplogEntry("proj_path_to_hash_same"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_path_to_hash_same"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + // Second op: same FileId transitions to CAS hash (same data) + { + CbPackage Op = BuildHashReferencedFileOp(SecondOpKey, MatchingBlob); + PostOplogEntry("proj_path_to_hash_same"sv, "oplog"sv, Op); + } + + if (RunGcAfterTransition) + { + TriggerGcAndWait(); + } + + // Must still resolve successfully after the transition + HttpClient::Response Response = GetChunk("proj_path_to_hash_same"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK_EQ(Payload.GetSize(), FileBlob.GetSize()); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } +} + TEST_SUITE_END(); } // namespace zen::tests -- cgit v1.2.3