From c32b6042dee8444f4e214f227005a657ec87531e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 27 Feb 2026 21:22:00 +0100 Subject: add multirange requests to blob store (#795) * add multirange requests to blob store --- src/zenserver-test/buildstore-tests.cpp | 200 +++++++++++++++++++++++++++++++- 1 file changed, 199 insertions(+), 1 deletion(-) (limited to 'src/zenserver-test/buildstore-tests.cpp') diff --git a/src/zenserver-test/buildstore-tests.cpp b/src/zenserver-test/buildstore-tests.cpp index 02b308485..ef48b2362 100644 --- a/src/zenserver-test/buildstore-tests.cpp +++ b/src/zenserver-test/buildstore-tests.cpp @@ -36,7 +36,8 @@ TEST_CASE("buildstore.blobs") std::string_view Bucket = "bkt"sv; Oid BuildId = Oid::NewOid(); - std::vector CompressedBlobsHashes; + std::vector CompressedBlobsHashes; + std::vector CompressedBlobsSizes; { ZenServerInstance Instance(TestEnv); @@ -51,6 +52,7 @@ TEST_CASE("buildstore.blobs") IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + CompressedBlobsSizes.push_back(CompressedBlob.GetCompressedSize()); IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); Payload.SetContentType(ZenContentType::kCompressedBinary); @@ -107,6 +109,7 @@ TEST_CASE("buildstore.blobs") IoBuffer Blob = CreateSemiRandomBlob(5713 + I * 7); CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + CompressedBlobsSizes.push_back(CompressedBlob.GetCompressedSize()); IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); Payload.SetContentType(ZenContentType::kCompressedBinary); @@ -141,6 +144,201 @@ TEST_CASE("buildstore.blobs") CHECK(IoHash::HashBuffer(Decompressed) == RawHash); } } + + { + // Single-range Get + + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + { + const IoHash& RawHash = CompressedBlobsHashes.front(); + uint64_t BlobSize = CompressedBlobsSizes.front(); + + std::vector> Ranges = {{BlobSize / 16 * 1, BlobSize / 2}}; + + uint64_t RangeSizeSum = Ranges.front().second; + + HttpClient::KeyValueMap Headers; + + Headers.Entries.insert( + {"Range", fmt::format("bytes={}-{}", Ranges.front().first, Ranges.front().first + Ranges.front().second - 1)}); + + HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), Headers); + REQUIRE(Result); + IoBuffer Payload = Result.ResponsePayload; + CHECK_EQ(RangeSizeSum, Payload.GetSize()); + + HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + HttpClient::Accept(ZenContentType::kCompressedBinary)); + REQUIRE(FullBlobResult); + MemoryView ActualRange = FullBlobResult.ResponsePayload.GetView().Mid(Ranges.front().first, Ranges.front().second); + MemoryView RangeView = Payload.GetView(); + CHECK(ActualRange.EqualBytes(RangeView)); + } + } + + { + // Single-range Post + + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + { + uint64_t RangeSizeSum = 0; + + const IoHash& RawHash = CompressedBlobsHashes.front(); + uint64_t BlobSize = CompressedBlobsSizes.front(); + + std::vector> Ranges = {{BlobSize / 16 * 1, BlobSize / 2}}; + + CbObjectWriter Writer; + Writer.BeginArray("ranges"sv); + { + for (const std::pair& Range : Ranges) + { + Writer.BeginObject(); + { + Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("length"sv, Range.second); + RangeSizeSum += Range.second; + } + Writer.EndObject(); + } + } + Writer.EndArray(); // ranges + + HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + Writer.Save(), + HttpClient::Accept(ZenContentType::kCbPackage)); + REQUIRE(Result); + IoBuffer Payload = Result.ResponsePayload; + REQUIRE(Payload.GetContentType() == ZenContentType::kCbPackage); + + CbPackage ResponsePackage = ParsePackageMessage(Payload); + CbObjectView ResponseObject = ResponsePackage.GetObject(); + + CbArrayView RangeArray = ResponseObject["ranges"sv].AsArrayView(); + CHECK_EQ(RangeArray.Num(), Ranges.size()); + size_t RangeOffset = 0; + for (CbFieldView View : RangeArray) + { + CbObjectView Range = View.AsObjectView(); + CHECK_EQ(Range["offset"sv].AsUInt64(), Ranges[RangeOffset].first); + CHECK_EQ(Range["length"sv].AsUInt64(), Ranges[RangeOffset].second); + RangeOffset++; + } + + const CbAttachment* DataAttachment = ResponsePackage.FindAttachment(RawHash); + REQUIRE(DataAttachment); + SharedBuffer PayloadRanges = DataAttachment->AsBinary(); + CHECK_EQ(RangeSizeSum, PayloadRanges.GetSize()); + + HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + HttpClient::Accept(ZenContentType::kCompressedBinary)); + REQUIRE(FullBlobResult); + + uint64_t Offset = 0; + for (const std::pair& Range : Ranges) + { + MemoryView ActualRange = FullBlobResult.ResponsePayload.GetView().Mid(Range.first, Range.second); + MemoryView RangeView = PayloadRanges.GetView().Mid(Offset, Range.second); + CHECK(ActualRange.EqualBytes(RangeView)); + Offset += Range.second; + } + } + } + + { + // Multi-range + + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + { + uint64_t RangeSizeSum = 0; + + const IoHash& RawHash = CompressedBlobsHashes.front(); + uint64_t BlobSize = CompressedBlobsSizes.front(); + + std::vector> Ranges = { + {BlobSize / 16 * 1, BlobSize / 20}, + {BlobSize / 16 * 3, BlobSize / 32}, + {BlobSize / 16 * 5, BlobSize / 16}, + {BlobSize - BlobSize / 16, BlobSize / 16 - 1}, + }; + + CbObjectWriter Writer; + Writer.BeginArray("ranges"sv); + { + for (const std::pair& Range : Ranges) + { + Writer.BeginObject(); + { + Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("length"sv, Range.second); + RangeSizeSum += Range.second; + } + Writer.EndObject(); + } + } + Writer.EndArray(); // ranges + + HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + Writer.Save(), + HttpClient::Accept(ZenContentType::kCbPackage)); + REQUIRE(Result); + IoBuffer Payload = Result.ResponsePayload; + REQUIRE(Payload.GetContentType() == ZenContentType::kCbPackage); + + CbPackage ResponsePackage = ParsePackageMessage(Payload); + CbObjectView ResponseObject = ResponsePackage.GetObject(); + + CbArrayView RangeArray = ResponseObject["ranges"sv].AsArrayView(); + CHECK_EQ(RangeArray.Num(), Ranges.size()); + size_t RangeOffset = 0; + for (CbFieldView View : RangeArray) + { + CbObjectView Range = View.AsObjectView(); + CHECK_EQ(Range["offset"sv].AsUInt64(), Ranges[RangeOffset].first); + CHECK_EQ(Range["length"sv].AsUInt64(), Ranges[RangeOffset].second); + RangeOffset++; + } + + const CbAttachment* DataAttachment = ResponsePackage.FindAttachment(RawHash); + REQUIRE(DataAttachment); + SharedBuffer PayloadRanges = DataAttachment->AsBinary(); + CHECK_EQ(RangeSizeSum, PayloadRanges.GetSize()); + + HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + HttpClient::Accept(ZenContentType::kCompressedBinary)); + REQUIRE(FullBlobResult); + + uint64_t Offset = 0; + for (const std::pair& Range : Ranges) + { + MemoryView ActualRange = FullBlobResult.ResponsePayload.GetView().Mid(Range.first, Range.second); + MemoryView RangeView = PayloadRanges.GetView().Mid(Offset, Range.second); + CHECK(ActualRange.EqualBytes(RangeView)); + Offset += Range.second; + } + } + } } namespace { -- cgit v1.2.3 From c7e0efb9c12f4607d4bc6a844a3e5bd3272bd839 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 28 Feb 2026 15:36:13 +0100 Subject: test running / reporting improvements (#797) **CI/CD improvements (validate.yml):** - Add test reporter (`ue-foundation/test-reporter@v2`) for all three platforms, rendering JUnit test results directly in PR check runs - Add "Trust workspace" step on Windows to fix git safe.directory ownership issue with self-hosted runners - Clean stale report files before each test run to prevent false failures from leftover XML - Broaden `paths-ignore` to skip builds for non-code changes (`*.md`, `LICENSE`, `.gitignore`, `docs/**`) **Test improvements:** - Convert `CHECK` to `REQUIRE` in several test suites (projectstore, integration, http) for fail-fast behavior - Mark some tests with `doctest::skip()` for selective execution - Skip httpclient transport tests pending investigation - Add `--noskip` option to `xmake test` task - Add `--repeat=` option to `xmake test` task, to run tests repeatedly N times or until there is a failure **xmake test output improvements:** - Add totals row to test summary table - Right-justify numeric columns in summary table --- src/zenserver-test/buildstore-tests.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'src/zenserver-test/buildstore-tests.cpp') diff --git a/src/zenserver-test/buildstore-tests.cpp b/src/zenserver-test/buildstore-tests.cpp index ef48b2362..7cd31db06 100644 --- a/src/zenserver-test/buildstore-tests.cpp +++ b/src/zenserver-test/buildstore-tests.cpp @@ -389,7 +389,7 @@ TEST_CASE("buildstore.metadata") HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/getBlobMetadata", Namespace, Bucket, BuildId), Payload, HttpClient::Accept(ZenContentType::kCbObject)); - CHECK(Result); + REQUIRE(Result); std::vector ResultMetadatas; @@ -570,7 +570,7 @@ TEST_CASE("buildstore.cache") { std::vector Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount; I++) { CHECK(Exists[I].HasBody); @@ -609,7 +609,7 @@ TEST_CASE("buildstore.cache") { std::vector Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount; I++) { CHECK(Exists[I].HasBody); @@ -617,7 +617,7 @@ TEST_CASE("buildstore.cache") } std::vector FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); - CHECK_EQ(BlobCount, FetchedMetadatas.size()); + REQUIRE_EQ(BlobCount, FetchedMetadatas.size()); for (size_t I = 0; I < BlobCount; I++) { @@ -638,7 +638,7 @@ TEST_CASE("buildstore.cache") { std::vector Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount * 2; I++) { CHECK(Exists[I].HasBody); @@ -649,7 +649,7 @@ TEST_CASE("buildstore.cache") CHECK_EQ(BlobCount, MetaDatas.size()); std::vector FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); - CHECK_EQ(BlobCount, FetchedMetadatas.size()); + REQUIRE_EQ(BlobCount, FetchedMetadatas.size()); for (size_t I = 0; I < BlobCount; I++) { @@ -672,7 +672,7 @@ TEST_CASE("buildstore.cache") CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, GetTinyWorkerPool(EWorkloadType::Background))); std::vector Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount * 2; I++) { CHECK(Exists[I].HasBody); @@ -691,7 +691,7 @@ TEST_CASE("buildstore.cache") CHECK_EQ(BlobCount, MetaDatas.size()); std::vector FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); - CHECK_EQ(BlobCount, FetchedMetadatas.size()); + REQUIRE_EQ(BlobCount, FetchedMetadatas.size()); for (size_t I = 0; I < BlobCount; I++) { -- cgit v1.2.3 From d604351cb5dc3032a7cb8c84d6ad5f1480325e5c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 2 Mar 2026 09:37:14 +0100 Subject: Add test suites (#799) Makes all test cases part of a test suite. Test suites are named after the module and the name of the file containing the implementation of the test. * This allows for better and more predictable filtering of which test cases to run which should also be able to reduce the time CI spends in tests since it can filter on the tests for that particular module. Also improves `xmake test` behaviour: * instead of an explicit list of projects just enumerate the test projects which are available based on build system state * also introduces logic to avoid running `xmake config` unnecessarily which would invalidate the existing build and do lots of unnecessary work since dependencies were invalidated by the updated config * also invokes build only for the chosen test targets As a bonus, also adds `xmake sln --open` which allows opening IDE after generation of solution/xmake project is done. --- src/zenserver-test/buildstore-tests.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/zenserver-test/buildstore-tests.cpp') diff --git a/src/zenserver-test/buildstore-tests.cpp b/src/zenserver-test/buildstore-tests.cpp index 7cd31db06..cf9b10896 100644 --- a/src/zenserver-test/buildstore-tests.cpp +++ b/src/zenserver-test/buildstore-tests.cpp @@ -27,6 +27,8 @@ namespace zen::tests { using namespace std::literals; +TEST_SUITE_BEGIN("server.buildstore"); + TEST_CASE("buildstore.blobs") { std::filesystem::path SystemRootPath = TestEnv.CreateNewTestDir(); @@ -700,5 +702,7 @@ TEST_CASE("buildstore.cache") } } +TEST_SUITE_END(); + } // namespace zen::tests #endif -- cgit v1.2.3