diff options
| author | Stefan Boberg <[email protected]> | 2026-02-28 15:36:13 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2026-02-28 15:36:13 +0100 |
| commit | c7e0efb9c12f4607d4bc6a844a3e5bd3272bd839 (patch) | |
| tree | 47c7edc37dcd54f06be04b40f96e6edc24d7f4ab /src | |
| parent | add multirange requests to blob store (#795) (diff) | |
| download | zen-c7e0efb9c12f4607d4bc6a844a3e5bd3272bd839.tar.xz zen-c7e0efb9c12f4607d4bc6a844a3e5bd3272bd839.zip | |
test running / reporting improvements (#797)
**CI/CD improvements (validate.yml):**
- Add test reporter (`ue-foundation/test-reporter@v2`) for all three platforms, rendering JUnit test results directly in PR check runs
- Add "Trust workspace" step on Windows to fix git safe.directory ownership issue with self-hosted runners
- Clean stale report files before each test run to prevent false failures from leftover XML
- Broaden `paths-ignore` to skip builds for non-code changes (`*.md`, `LICENSE`, `.gitignore`, `docs/**`)
**Test improvements:**
- Convert `CHECK` to `REQUIRE` in several test suites (projectstore, integration, http) for fail-fast behavior
- Mark some tests with `doctest::skip()` for selective execution
- Skip httpclient transport tests pending investigation
- Add `--noskip` option to `xmake test` task
- Add `--repeat=<N>` option to `xmake test` task, to run tests repeatedly N times or until there is a failure
**xmake test output improvements:**
- Add totals row to test summary table
- Right-justify numeric columns in summary table
Diffstat (limited to 'src')
| -rw-r--r-- | src/zenhttp/httpclient_test.cpp | 4 | ||||
| -rw-r--r-- | src/zenserver-test/buildstore-tests.cpp | 16 | ||||
| -rw-r--r-- | src/zenserver-test/cache-tests.cpp | 10 | ||||
| -rw-r--r-- | src/zenserver-test/hub-tests.cpp | 2 | ||||
| -rw-r--r-- | src/zenserver-test/projectstore-tests.cpp | 34 | ||||
| -rw-r--r-- | src/zenserver-test/workspace-tests.cpp | 4 | ||||
| -rw-r--r-- | src/zenstore/cache/structuredcachestore.cpp | 2 |
7 files changed, 33 insertions, 39 deletions
diff --git a/src/zenhttp/httpclient_test.cpp b/src/zenhttp/httpclient_test.cpp index 509b56371..91b1a3414 100644 --- a/src/zenhttp/httpclient_test.cpp +++ b/src/zenhttp/httpclient_test.cpp @@ -1079,7 +1079,7 @@ struct FaultTcpServer } }; -TEST_CASE("httpclient.transport-faults") +TEST_CASE("httpclient.transport-faults" * doctest::skip()) { SUBCASE("connection reset before response") { @@ -1217,7 +1217,7 @@ TEST_CASE("httpclient.transport-faults") } } -TEST_CASE("httpclient.transport-faults-post") +TEST_CASE("httpclient.transport-faults-post" * doctest::skip()) { constexpr size_t kPostBodySize = 256 * 1024; diff --git a/src/zenserver-test/buildstore-tests.cpp b/src/zenserver-test/buildstore-tests.cpp index ef48b2362..7cd31db06 100644 --- a/src/zenserver-test/buildstore-tests.cpp +++ b/src/zenserver-test/buildstore-tests.cpp @@ -389,7 +389,7 @@ TEST_CASE("buildstore.metadata") HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/getBlobMetadata", Namespace, Bucket, BuildId), Payload, HttpClient::Accept(ZenContentType::kCbObject)); - CHECK(Result); + REQUIRE(Result); std::vector<CbObject> ResultMetadatas; @@ -570,7 +570,7 @@ TEST_CASE("buildstore.cache") { std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount; I++) { CHECK(Exists[I].HasBody); @@ -609,7 +609,7 @@ TEST_CASE("buildstore.cache") { std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount; I++) { CHECK(Exists[I].HasBody); @@ -617,7 +617,7 @@ TEST_CASE("buildstore.cache") } std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); - CHECK_EQ(BlobCount, FetchedMetadatas.size()); + REQUIRE_EQ(BlobCount, FetchedMetadatas.size()); for (size_t I = 0; I < BlobCount; I++) { @@ -638,7 +638,7 @@ TEST_CASE("buildstore.cache") { std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount * 2; I++) { CHECK(Exists[I].HasBody); @@ -649,7 +649,7 @@ TEST_CASE("buildstore.cache") CHECK_EQ(BlobCount, MetaDatas.size()); std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); - CHECK_EQ(BlobCount, FetchedMetadatas.size()); + REQUIRE_EQ(BlobCount, FetchedMetadatas.size()); for (size_t I = 0; I < BlobCount; I++) { @@ -672,7 +672,7 @@ TEST_CASE("buildstore.cache") CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, GetTinyWorkerPool(EWorkloadType::Background))); std::vector<BuildStorageCache::BlobExistsResult> Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount * 2; I++) { CHECK(Exists[I].HasBody); @@ -691,7 +691,7 @@ TEST_CASE("buildstore.cache") CHECK_EQ(BlobCount, MetaDatas.size()); std::vector<CbObject> FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); - CHECK_EQ(BlobCount, FetchedMetadatas.size()); + REQUIRE_EQ(BlobCount, FetchedMetadatas.size()); for (size_t I = 0; I < BlobCount; I++) { diff --git a/src/zenserver-test/cache-tests.cpp b/src/zenserver-test/cache-tests.cpp index 0272d3797..745a89253 100644 --- a/src/zenserver-test/cache-tests.cpp +++ b/src/zenserver-test/cache-tests.cpp @@ -145,7 +145,7 @@ TEST_CASE("zcache.cbpackage") for (const zen::CbAttachment& LhsAttachment : LhsAttachments) { const zen::CbAttachment* RhsAttachment = Rhs.FindAttachment(LhsAttachment.GetHash()); - CHECK(RhsAttachment); + REQUIRE(RhsAttachment); zen::SharedBuffer LhsBuffer = LhsAttachment.AsCompressedBinary().Decompress(); CHECK(!LhsBuffer.IsNull()); @@ -1373,14 +1373,8 @@ TEST_CASE("zcache.rpc") } } -TEST_CASE("zcache.failing.upstream") +TEST_CASE("zcache.failing.upstream" * doctest::skip()) { - // This is an exploratory test that takes a long time to run, so lets skip it by default - if (true) - { - return; - } - using namespace std::literals; using namespace utils; diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp index 42a5dcae4..bd85a5020 100644 --- a/src/zenserver-test/hub-tests.cpp +++ b/src/zenserver-test/hub-tests.cpp @@ -232,7 +232,7 @@ TEST_CASE("hub.lifecycle.children") TEST_SUITE_END(); -TEST_CASE("hub.consul.lifecycle") +TEST_CASE("hub.consul.lifecycle" * doctest::skip()) { zen::consul::ConsulProcess ConsulProc; ConsulProc.SpawnConsulAgent(); diff --git a/src/zenserver-test/projectstore-tests.cpp b/src/zenserver-test/projectstore-tests.cpp index 735aef159..487832405 100644 --- a/src/zenserver-test/projectstore-tests.cpp +++ b/src/zenserver-test/projectstore-tests.cpp @@ -71,7 +71,7 @@ TEST_CASE("project.basic") { auto Response = Http.Get("/prj/test"sv); - CHECK(Response.StatusCode == HttpResponseCode::OK); + REQUIRE(Response.StatusCode == HttpResponseCode::OK); CbObject ResponseObject = Response.AsObject(); @@ -92,7 +92,7 @@ TEST_CASE("project.basic") { auto Response = Http.Get(""sv); - CHECK(Response.StatusCode == HttpResponseCode::OK); + REQUIRE(Response.StatusCode == HttpResponseCode::OK); CbObject ResponseObject = Response.AsObject(); @@ -213,7 +213,7 @@ TEST_CASE("project.basic") auto Response = Http.Get(ChunkGetUri); REQUIRE(Response); - CHECK(Response.StatusCode == HttpResponseCode::OK); + REQUIRE(Response.StatusCode == HttpResponseCode::OK); IoBuffer Data = Response.ResponsePayload; IoBuffer ReferenceData = IoBufferBuilder::MakeFromFile(RootPath / BinPath); @@ -235,13 +235,13 @@ TEST_CASE("project.basic") auto Response = Http.Get(ChunkGetUri, {{"Accept-Type", "application/x-ue-comp"}}); REQUIRE(Response); - CHECK(Response.StatusCode == HttpResponseCode::OK); + REQUIRE(Response.StatusCode == HttpResponseCode::OK); IoBuffer Data = Response.ResponsePayload; IoHash RawHash; uint64_t RawSize; CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Data), RawHash, RawSize); - CHECK(Compressed); + REQUIRE(Compressed); IoBuffer DataDecompressed = Compressed.Decompress().AsIoBuffer(); IoBuffer ReferenceData = IoBufferBuilder::MakeFromFile(RootPath / BinPath); CHECK(RawSize == ReferenceData.GetSize()); @@ -436,13 +436,13 @@ TEST_CASE("project.remote") HttpClient Http{UrlBase}; HttpClient::Response Response = Http.Post(fmt::format("/prj/{}", ProjectName), ProjectPayload); - CHECK(Response); + REQUIRE(Response); }; auto MakeOplog = [](std::string_view UrlBase, std::string_view ProjectName, std::string_view OplogName) { HttpClient Http{UrlBase}; HttpClient::Response Response = Http.Post(fmt::format("/prj/{}/oplog/{}", ProjectName, OplogName), IoBuffer{}); - CHECK(Response); + REQUIRE(Response); }; auto MakeOp = [](std::string_view UrlBase, std::string_view ProjectName, std::string_view OplogName, const CbPackage& OpPackage) { @@ -453,7 +453,7 @@ TEST_CASE("project.remote") HttpClient Http{UrlBase}; HttpClient::Response Response = Http.Post(fmt::format("/prj/{}/oplog/{}/new", ProjectName, OplogName), Body); - CHECK(Response); + REQUIRE(Response); }; MakeProject(Servers.GetInstance(0).GetBaseUri(), "proj0"); @@ -504,7 +504,7 @@ TEST_CASE("project.remote") HttpClient::Response Response = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", Project, Oplog), Payload, {{"Accept", "application/x-ue-cbpkg"}}); - CHECK(Response); + REQUIRE(Response); CbPackage ResponsePackage = ParsePackageMessage(Response.ResponsePayload); CHECK(ResponsePackage.GetAttachments().size() == AttachmentHashes.size()); for (auto A : ResponsePackage.GetAttachments()) @@ -519,7 +519,7 @@ TEST_CASE("project.remote") HttpClient Http{Servers.GetInstance(ServerIndex).GetBaseUri()}; HttpClient::Response Response = Http.Get(fmt::format("/prj/{}/oplog/{}/entries", Project, Oplog)); - CHECK(Response); + REQUIRE(Response); IoBuffer Payload(Response.ResponsePayload); CbObject OplogResonse = LoadCompactBinaryObject(Payload); @@ -541,7 +541,7 @@ TEST_CASE("project.remote") auto HttpWaitForCompletion = [](ZenServerInstance& Server, const HttpClient::Response& Response) { REQUIRE(Response); const uint64_t JobId = ParseInt<uint64_t>(Response.AsText()).value_or(0); - CHECK(JobId != 0); + REQUIRE(JobId != 0); HttpClient Http{Server.GetBaseUri()}; @@ -549,10 +549,10 @@ TEST_CASE("project.remote") { HttpClient::Response StatusResponse = Http.Get(fmt::format("/admin/jobs/{}", JobId), {{"Accept", ToString(ZenContentType::kCbObject)}}); - CHECK(StatusResponse); + REQUIRE(StatusResponse); CbObject ResponseObject = StatusResponse.AsObject(); std::string_view Status = ResponseObject["Status"sv].AsString(); - CHECK(Status != "Aborted"sv); + REQUIRE(Status != "Aborted"sv); if (Status == "Complete"sv) { return; @@ -887,16 +887,16 @@ TEST_CASE("project.rpcappendop") Project.AddString("project"sv, ""sv); Project.AddString("projectfile"sv, ""sv); HttpClient::Response Response = Client.Post(fmt::format("/prj/{}", ProjectName), Project.Save()); - CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); }; auto MakeOplog = [](HttpClient& Client, std::string_view ProjectName, std::string_view OplogName) { HttpClient::Response Response = Client.Post(fmt::format("/prj/{}/oplog/{}", ProjectName, OplogName)); - CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); }; auto GetOplog = [](HttpClient& Client, std::string_view ProjectName, std::string_view OplogName) { HttpClient::Response Response = Client.Get(fmt::format("/prj/{}/oplog/{}", ProjectName, OplogName)); - CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); return Response.AsObject(); }; @@ -910,7 +910,7 @@ TEST_CASE("project.rpcappendop") } Request.EndArray(); // "ops" HttpClient::Response Response = Client.Post(fmt::format("/prj/{}/oplog/{}/rpc", ProjectName, OplogName), Request.Save()); - CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); CbObjectView ResponsePayload = Response.AsPackage().GetObject(); CbArrayView NeedArray = ResponsePayload["need"sv].AsArrayView(); diff --git a/src/zenserver-test/workspace-tests.cpp b/src/zenserver-test/workspace-tests.cpp index 7595d790a..aedadf0c3 100644 --- a/src/zenserver-test/workspace-tests.cpp +++ b/src/zenserver-test/workspace-tests.cpp @@ -514,9 +514,9 @@ TEST_CASE("workspaces.share") } IoBuffer BatchResponse = Client.Post(fmt::format("/ws/{}/{}/batch", WorkspaceId, ShareId), BuildChunkBatchRequest(BatchEntries)).ResponsePayload; - CHECK(BatchResponse); + REQUIRE(BatchResponse); std::vector<IoBuffer> BatchResult = ParseChunkBatchResponse(BatchResponse); - CHECK(BatchResult.size() == Files.size()); + REQUIRE(BatchResult.size() == Files.size()); for (const RequestChunkEntry& Request : BatchEntries) { IoBuffer Result = BatchResult[Request.CorrelationId]; diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 4e8475293..d8a5755c5 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -1551,7 +1551,7 @@ TEST_CASE("cachestore.size") } } -TEST_CASE("cachestore.threadedinsert") // * doctest::skip(true)) +TEST_CASE("cachestore.threadedinsert" * doctest::skip()) { // for (uint32_t i = 0; i < 100; ++i) { |