diff options
Diffstat (limited to 'src/zenserver-test/projectstore-tests.cpp')
| -rw-r--r-- | src/zenserver-test/projectstore-tests.cpp | 516 |
1 files changed, 510 insertions, 6 deletions
diff --git a/src/zenserver-test/projectstore-tests.cpp b/src/zenserver-test/projectstore-tests.cpp index 5cc75c590..cec453511 100644 --- a/src/zenserver-test/projectstore-tests.cpp +++ b/src/zenserver-test/projectstore-tests.cpp @@ -22,6 +22,7 @@ ZEN_THIRD_PARTY_INCLUDES_START ZEN_THIRD_PARTY_INCLUDES_END # include <random> +# include <thread> namespace zen::tests { @@ -340,6 +341,102 @@ TEST_CASE("project.basic") ZEN_INFO("+++++++"); } + SUBCASE("snapshot zero byte file") + { + // A zero-byte file referenced in an oplog entry must survive a + // snapshot: the file is read, compressed, stored in CidStore, and + // the oplog is rewritten with a BinaryAttachment reference. After + // the snapshot the chunk must be retrievable and decompress to an + // empty payload. + + std::filesystem::path EmptyFileRelPath = std::filesystem::path("zerobyte_snapshot_test") / "empty.bin"; + std::filesystem::path EmptyFileAbsPath = RootPath / EmptyFileRelPath; + CreateDirectories(MakeSafeAbsolutePath(EmptyFileAbsPath.parent_path())); + // Create a zero-byte file on disk. + WriteFile(MakeSafeAbsolutePath(EmptyFileAbsPath), IoBuffer{}); + REQUIRE(IsFile(MakeSafeAbsolutePath(EmptyFileAbsPath))); + + const std::string_view EmptyChunkId{ + "00000000" + "00000000" + "00030000"}; + auto EmptyFileOid = zen::Oid::FromHexString(EmptyChunkId); + + zen::CbObjectWriter OpWriter; + OpWriter << "key" + << "zero_byte_test"; + OpWriter.BeginArray("files"); + OpWriter.BeginObject(); + OpWriter << "id" << EmptyFileOid; + OpWriter << "clientpath" + << "/{engine}/empty_file"; + OpWriter << "serverpath" << EmptyFileRelPath.c_str(); + OpWriter.EndObject(); + OpWriter.EndArray(); + + zen::CbObject Op = OpWriter.Save(); + zen::CbPackage OpPackage(Op); + + zen::BinaryWriter MemOut; + legacy::SaveCbPackage(OpPackage, MemOut); + + HttpClient Http{BaseUri}; + + { + auto Response = Http.Post("/new", IoBufferBuilder::MakeFromMemory(MemOut.GetView())); + REQUIRE(Response); + CHECK(Response.StatusCode == HttpResponseCode::Created); + } + + // Read file data before snapshot - raw and uncompressed, 0 bytes. + // http.sys converts a 200 OK with empty body to 204 No Content, so + // accept either status code. + { + zen::StringBuilder<128> ChunkGetUri; + ChunkGetUri << "/" << EmptyChunkId; + auto Response = Http.Get(ChunkGetUri); + + REQUIRE(Response); + CHECK((Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::NoContent)); + CHECK(Response.ResponsePayload.GetSize() == 0); + } + + // Trigger snapshot. + { + IoBuffer Payload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { Writer.AddString("method"sv, "snapshot"sv); }); + auto Response = Http.Post("/rpc"sv, Payload); + REQUIRE(Response); + CHECK(Response.StatusCode == HttpResponseCode::OK); + } + + // Read chunk after snapshot - compressed, decompresses to 0 bytes. + { + zen::StringBuilder<128> ChunkGetUri; + ChunkGetUri << "/" << EmptyChunkId; + auto Response = Http.Get(ChunkGetUri, {{"Accept-Type", "application/x-ue-comp"}}); + + REQUIRE(Response); + REQUIRE(Response.StatusCode == HttpResponseCode::OK); + + IoBuffer Data = Response.ResponsePayload; + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Data), RawHash, RawSize); + REQUIRE(Compressed); + CHECK(RawSize == 0); + IoBuffer DataDecompressed = Compressed.Decompress().AsIoBuffer(); + CHECK(DataDecompressed.GetSize() == 0); + } + + // Cleanup + { + std::error_code Ec; + DeleteDirectories(MakeSafeAbsolutePath(RootPath / "zerobyte_snapshot_test"), Ec); + } + + ZEN_INFO("+++++++"); + } + SUBCASE("test chunk not found error") { HttpClient Http{BaseUri}; @@ -1027,7 +1124,7 @@ TEST_CASE("project.rpcappendop") std::string_view ProjectName, std::string_view OplogName, std::span<const CompressedBuffer> Attachments, - void* ServerProcessHandle, + const ProcessHandle& ServerProcessHandle, const std::filesystem::path& TempPath) { CompositeBuffer PackageMessage; { @@ -1054,7 +1151,8 @@ TEST_CASE("project.rpcappendop") Request.EndArray(); // "chunks" RequestPackage.SetObject(Request.Save()); - PackageMessage = CompositeBuffer(FormatPackageMessage(RequestPackage, FormatFlags::kAllowLocalReferences, ServerProcessHandle)); + PackageMessage = + CompositeBuffer(FormatPackageMessage(RequestPackage, FormatFlags::kAllowLocalReferences, ServerProcessHandle.Handle())); } HttpClient::Response Response = @@ -1063,8 +1161,8 @@ TEST_CASE("project.rpcappendop") }; { - HttpClient Client(Servers.GetInstance(0).GetBaseUri()); - void* ServerProcessHandle = Servers.GetInstance(0).GetProcessHandle(); + HttpClient Client(Servers.GetInstance(0).GetBaseUri()); + const ProcessHandle& ServerProcessHandle = Servers.GetInstance(0).GetProcessHandle(); MakeProject(Client, "proj0"); MakeOplog(Client, "proj0", "oplog0"); @@ -1108,8 +1206,8 @@ TEST_CASE("project.rpcappendop") } { - HttpClient Client(Servers.GetInstance(1).GetBaseUri()); - void* ServerProcessHandle = nullptr; // Force use of path for attachments passed on disk + HttpClient Client(Servers.GetInstance(1).GetBaseUri()); + ProcessHandle ServerProcessHandle; // Force use of path for attachments passed on disk MakeProject(Client, "proj0"); MakeOplog(Client, "proj0", "oplog0"); @@ -1153,6 +1251,412 @@ TEST_CASE("project.rpcappendop") } } +TEST_CASE("project.file.data.transitions") +{ + using namespace utils; + + std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(); + + zen::StringBuilder<64> ServerBaseUri; + ServerBaseUri << fmt::format("http://localhost:{}", PortNumber); + + // Set up a root directory with a test file on disk for path-referenced serving + std::filesystem::path RootDir = TestDir / "root"; + std::filesystem::path TestFilePath = RootDir / "content" / "testfile.bin"; + std::filesystem::path RelServerPath = std::filesystem::path("content") / "testfile.bin"; + CreateDirectories(TestFilePath.parent_path()); + IoBuffer FileBlob = CreateRandomBlob(4096); + WriteFile(TestFilePath, FileBlob); + + // Create a compressed blob to use as a CAS-referenced attachment (content differs from FileBlob) + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(2048))); + + // Fixed chunk IDs for the file entry across sub-tests + const std::string_view FileChunkIdStr{ + "aa000000" + "bb000000" + "cc000001"}; + Oid FileOid = Oid::FromHexString(FileChunkIdStr); + + HttpClient Http{ServerBaseUri}; + + auto MakeProject = [&](std::string_view ProjectName) { + CbObjectWriter Project; + Project.AddString("id"sv, ProjectName); + Project.AddString("root"sv, PathToUtf8(RootDir.c_str())); + Project.AddString("engine"sv, ""sv); + Project.AddString("project"sv, ""sv); + Project.AddString("projectfile"sv, ""sv); + HttpClient::Response Response = Http.Post(fmt::format("/prj/{}", ProjectName), Project.Save()); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("MakeProject")); + }; + + auto MakeOplog = [&](std::string_view ProjectName, std::string_view OplogName) { + HttpClient::Response Response = + Http.Post(fmt::format("/prj/{}/oplog/{}", ProjectName, OplogName), IoBuffer{}, ZenContentType::kCbObject); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("MakeOplog")); + }; + + auto PostOplogEntry = [&](std::string_view ProjectName, std::string_view OplogName, const CbPackage& OpPackage) { + zen::BinaryWriter MemOut; + legacy::SaveCbPackage(OpPackage, MemOut); + IoBuffer Body{IoBuffer::Wrap, MemOut.GetData(), MemOut.GetSize()}; + Body.SetContentType(HttpContentType::kCbPackage); + HttpClient::Response Response = Http.Post(fmt::format("/prj/{}/oplog/{}/new", ProjectName, OplogName), Body); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("PostOplogEntry")); + }; + + auto GetChunk = [&](std::string_view ProjectName) -> HttpClient::Response { + return Http.Get(fmt::format("/prj/{}/oplog/oplog/{}", ProjectName, FileChunkIdStr)); + }; + + // Extract the raw decompressed bytes from a chunk response, handling both compressed and uncompressed payloads + auto GetDecompressedPayload = [](const HttpClient::Response& Response) -> IoBuffer { + if (Response.ResponsePayload.GetContentType() == ZenContentType::kCompressedBinary) + { + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Response.ResponsePayload), RawHash, RawSize); + REQUIRE(Compressed); + return Compressed.Decompress().AsIoBuffer(); + } + return Response.ResponsePayload; + }; + + auto TriggerGcAndWait = [&]() { + HttpClient::Response TriggerResponse = Http.Post("/admin/gc?smallobjects=true"sv, IoBuffer{}); + REQUIRE_MESSAGE(TriggerResponse.IsSuccess(), TriggerResponse.ErrorMessage("TriggerGc")); + + for (int Attempt = 0; Attempt < 100; ++Attempt) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + HttpClient::Response StatusResponse = Http.Get("/admin/gc"sv); + REQUIRE_MESSAGE(StatusResponse.IsSuccess(), StatusResponse.ErrorMessage("GcStatus")); + CbObject StatusObj = StatusResponse.AsObject(); + if (StatusObj["Status"sv].AsString() == "Idle"sv) + { + return; + } + } + FAIL("GC did not complete within timeout"); + }; + + auto BuildPathReferencedFileOp = [&](const Oid& KeyId) -> CbPackage { + CbPackage Package; + CbObjectWriter Object; + Object << "key"sv << OidAsString(KeyId); + Object.BeginArray("files"sv); + Object.BeginObject(); + Object << "id"sv << FileOid; + Object << "serverpath"sv << RelServerPath.string(); + Object << "clientpath"sv + << "/{engine}/testfile.bin"sv; + Object.EndObject(); + Object.EndArray(); + Package.SetObject(Object.Save()); + return Package; + }; + + auto BuildHashReferencedFileOp = [&](const Oid& KeyId, const CompressedBuffer& Blob) -> CbPackage { + CbPackage Package; + CbObjectWriter Object; + Object << "key"sv << OidAsString(KeyId); + CbAttachment Attach(Blob, Blob.DecodeRawHash()); + Object.BeginArray("files"sv); + Object.BeginObject(); + Object << "id"sv << FileOid; + Object << "data"sv << Attach; + Object << "clientpath"sv + << "/{engine}/testfile.bin"sv; + Object.EndObject(); + Object.EndArray(); + Package.AddAttachment(Attach); + Package.SetObject(Object.Save()); + return Package; + }; + + SUBCASE("path-referenced file is retrievable") + { + MakeProject("proj_path"sv); + MakeOplog("proj_path"sv, "oplog"sv); + + CbPackage Op = BuildPathReferencedFileOp(Oid::NewOid()); + PostOplogEntry("proj_path"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_path"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK_EQ(Payload.GetSize(), FileBlob.GetSize()); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + SUBCASE("hash-referenced file is retrievable") + { + MakeProject("proj_hash"sv); + MakeOplog("proj_hash"sv, "oplog"sv); + + CbPackage Op = BuildHashReferencedFileOp(Oid::NewOid(), CompressedBlob); + PostOplogEntry("proj_hash"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_hash"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + IoBuffer ExpectedDecompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK_EQ(Payload.GetSize(), ExpectedDecompressed.GetSize()); + CHECK(Payload.GetView().EqualBytes(ExpectedDecompressed.GetView())); + } + } + + SUBCASE("hash-referenced to path-referenced transition with different content") + { + MakeProject("proj_hash_to_path_diff"sv); + MakeOplog("proj_hash_to_path_diff"sv, "oplog"sv); + + Oid FirstOpKey = Oid::NewOid(); + Oid SecondOpKey; + bool RunGcAfterTransition = false; + + SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); } + SUBCASE("same op key") { SecondOpKey = FirstOpKey; } + SUBCASE("new op key with gc") + { + SecondOpKey = Oid::NewOid(); + RunGcAfterTransition = true; + } + SUBCASE("same op key with gc") + { + SecondOpKey = FirstOpKey; + RunGcAfterTransition = true; + } + + // First op: file with CAS hash (content differs from the on-disk file) + { + CbPackage Op = BuildHashReferencedFileOp(FirstOpKey, CompressedBlob); + PostOplogEntry("proj_hash_to_path_diff"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_hash_to_path_diff"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + IoBuffer ExpectedDecompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(Payload.GetView().EqualBytes(ExpectedDecompressed.GetView())); + } + } + + // Second op: same FileId transitions to serverpath (different data) + { + CbPackage Op = BuildPathReferencedFileOp(SecondOpKey); + PostOplogEntry("proj_hash_to_path_diff"sv, "oplog"sv, Op); + } + + if (RunGcAfterTransition) + { + TriggerGcAndWait(); + } + + // Must serve the on-disk file content, not the old CAS blob + HttpClient::Response Response = GetChunk("proj_hash_to_path_diff"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK_EQ(Payload.GetSize(), FileBlob.GetSize()); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + SUBCASE("hash-referenced to path-referenced transition with identical content") + { + // Compress the same on-disk file content as a CAS blob so both references yield identical data + CompressedBuffer MatchingBlob = CompressedBuffer::Compress(SharedBuffer::Clone(FileBlob.GetView())); + + MakeProject("proj_hash_to_path_same"sv); + MakeOplog("proj_hash_to_path_same"sv, "oplog"sv); + + Oid FirstOpKey = Oid::NewOid(); + Oid SecondOpKey; + bool RunGcAfterTransition = false; + + SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); } + SUBCASE("same op key") { SecondOpKey = FirstOpKey; } + SUBCASE("new op key with gc") + { + SecondOpKey = Oid::NewOid(); + RunGcAfterTransition = true; + } + SUBCASE("same op key with gc") + { + SecondOpKey = FirstOpKey; + RunGcAfterTransition = true; + } + + // First op: file with CAS hash (content matches the on-disk file) + { + CbPackage Op = BuildHashReferencedFileOp(FirstOpKey, MatchingBlob); + PostOplogEntry("proj_hash_to_path_same"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_hash_to_path_same"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + // Second op: same FileId transitions to serverpath (same data) + { + CbPackage Op = BuildPathReferencedFileOp(SecondOpKey); + PostOplogEntry("proj_hash_to_path_same"sv, "oplog"sv, Op); + } + + if (RunGcAfterTransition) + { + TriggerGcAndWait(); + } + + // Must still resolve successfully after the transition + HttpClient::Response Response = GetChunk("proj_hash_to_path_same"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK_EQ(Payload.GetSize(), FileBlob.GetSize()); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + SUBCASE("path-referenced to hash-referenced transition with different content") + { + MakeProject("proj_path_to_hash_diff"sv); + MakeOplog("proj_path_to_hash_diff"sv, "oplog"sv); + + Oid FirstOpKey = Oid::NewOid(); + Oid SecondOpKey; + bool RunGcAfterTransition = false; + + SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); } + SUBCASE("same op key") { SecondOpKey = FirstOpKey; } + SUBCASE("new op key with gc") + { + SecondOpKey = Oid::NewOid(); + RunGcAfterTransition = true; + } + SUBCASE("same op key with gc") + { + SecondOpKey = FirstOpKey; + RunGcAfterTransition = true; + } + + // First op: file with serverpath + { + CbPackage Op = BuildPathReferencedFileOp(FirstOpKey); + PostOplogEntry("proj_path_to_hash_diff"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_path_to_hash_diff"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + // Second op: same FileId transitions to CAS hash (different data) + { + CbPackage Op = BuildHashReferencedFileOp(SecondOpKey, CompressedBlob); + PostOplogEntry("proj_path_to_hash_diff"sv, "oplog"sv, Op); + } + + if (RunGcAfterTransition) + { + TriggerGcAndWait(); + } + + // Must serve the CAS blob content, not the old on-disk file + HttpClient::Response Response = GetChunk("proj_path_to_hash_diff"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + IoBuffer ExpectedDecompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK_EQ(Payload.GetSize(), ExpectedDecompressed.GetSize()); + CHECK(Payload.GetView().EqualBytes(ExpectedDecompressed.GetView())); + } + } + + SUBCASE("path-referenced to hash-referenced transition with identical content") + { + // Compress the same on-disk file content as a CAS blob so both references yield identical data + CompressedBuffer MatchingBlob = CompressedBuffer::Compress(SharedBuffer::Clone(FileBlob.GetView())); + + MakeProject("proj_path_to_hash_same"sv); + MakeOplog("proj_path_to_hash_same"sv, "oplog"sv); + + Oid FirstOpKey = Oid::NewOid(); + Oid SecondOpKey; + bool RunGcAfterTransition = false; + + SUBCASE("new op key") { SecondOpKey = Oid::NewOid(); } + SUBCASE("same op key") { SecondOpKey = FirstOpKey; } + SUBCASE("new op key with gc") + { + SecondOpKey = Oid::NewOid(); + RunGcAfterTransition = true; + } + SUBCASE("same op key with gc") + { + SecondOpKey = FirstOpKey; + RunGcAfterTransition = true; + } + + // First op: file with serverpath + { + CbPackage Op = BuildPathReferencedFileOp(FirstOpKey); + PostOplogEntry("proj_path_to_hash_same"sv, "oplog"sv, Op); + + HttpClient::Response Response = GetChunk("proj_path_to_hash_same"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk first op")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } + + // Second op: same FileId transitions to CAS hash (same data) + { + CbPackage Op = BuildHashReferencedFileOp(SecondOpKey, MatchingBlob); + PostOplogEntry("proj_path_to_hash_same"sv, "oplog"sv, Op); + } + + if (RunGcAfterTransition) + { + TriggerGcAndWait(); + } + + // Must still resolve successfully after the transition + HttpClient::Response Response = GetChunk("proj_path_to_hash_same"sv); + CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("GetChunk after transition")); + if (Response.IsSuccess()) + { + IoBuffer Payload = GetDecompressedPayload(Response); + CHECK_EQ(Payload.GetSize(), FileBlob.GetSize()); + CHECK(Payload.GetView().EqualBytes(FileBlob.GetView())); + } + } +} + TEST_SUITE_END(); } // namespace zen::tests |