From 1ed3139e577f6c8aa6d07f7e76afa3a80d9d4852 Mon Sep 17 00:00:00 2001
From: Stefan Boberg <stefan.boberg@epicgames.com>
Date: Fri, 27 Feb 2026 19:36:22 +0100
Subject: Add test summary table and failure reporting to xmake test (#794)

- Add a summary table printed after all test suites complete, showing per-suite test case counts, assertion counts, timings and pass/fail status.
- Add failure reporting: individual failing test cases are listed at the end with their file path and line number for easy navigation.
- Made zenserver instances spawned by a hub not create new console windows for a better background testing experience
- The TestListener in testing.cpp now writes a machine-readable summary file (via `ZEN_TEST_SUMMARY_FILE` env var) containing aggregate counts and per-test-case failure details. This runs as a doctest listener alongside any active reporter, so it works with both console and JUnit modes.
- Tests now run in a deterministic order defined by a single ordered list that also serves as the test name/target mapping, replacing the previous unordered table + separate order list.
- The `--run` option now accepts comma-separated values  (e.g. `--run=core,http,util`) and validates each name, reporting unknown test names early.
- Fix platform detection in `xmake test`: the config command now passes `-p` explicitly, fixing "mingw" misdetection when running from Git Bash on Windows.
- Add missing "util" entry to the help text for `--run`.
---
 src/zencore/testing.cpp | 56 +++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 52 insertions(+), 4 deletions(-)

(limited to 'src/zencore/testing.cpp')

diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp
index 936424e0f..ef8fb0480 100644
--- a/src/zencore/testing.cpp
+++ b/src/zencore/testing.cpp
@@ -5,6 +5,12 @@
 
 #if ZEN_WITH_TESTS
 
+#	include <chrono>
+#	include <cstdlib>
+#	include <cstdio>
+#	include <string>
+#	include <vector>
+
 #	include <doctest/doctest.h>
 
 namespace zen::testing {
@@ -21,9 +27,35 @@ struct TestListener : public doctest::IReporter
 
 	void report_query(const doctest::QueryData& /*in*/) override {}
 
-	void test_run_start() override {}
+	void test_run_start() override { RunStart = std::chrono::steady_clock::now(); }
+
+	void test_run_end(const doctest::TestRunStats& in) override
+	{
+		auto   elapsed		  = std::chrono::steady_clock::now() - RunStart;
+		double elapsedSeconds = std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() / 1000.0;
 
-	void test_run_end(const doctest::TestRunStats& /*in*/) override {}
+		// Write machine-readable summary to file if requested (used by xmake test summary table)
+		const char* summaryFile = std::getenv("ZEN_TEST_SUMMARY_FILE");
+		if (summaryFile && summaryFile[0] != '\0')
+		{
+			if (FILE* f = std::fopen(summaryFile, "w"))
+			{
+				std::fprintf(f,
+							 "cases_total=%u\ncases_passed=%u\nassertions_total=%d\nassertions_passed=%d\n"
+							 "elapsed_seconds=%.3f\n",
+							 in.numTestCasesPassingFilters,
+							 in.numTestCasesPassingFilters - in.numTestCasesFailed,
+							 in.numAsserts,
+							 in.numAsserts - in.numAssertsFailed,
+							 elapsedSeconds);
+				for (const auto& failure : FailedTests)
+				{
+					std::fprintf(f, "failed=%s|%s|%u\n", failure.Name.c_str(), failure.File.c_str(), failure.Line);
+				}
+				std::fclose(f);
+			}
+		}
+	}
 
 	void test_case_start(const doctest::TestCaseData& in) override
 	{
@@ -37,7 +69,14 @@ struct TestListener : public doctest::IReporter
 		ZEN_CONSOLE("{}-------------------------------------------------------------------------------{}", ColorYellow, ColorNone);
 	}
 
-	void test_case_end(const doctest::CurrentTestCaseStats& /*in*/) override { Current = nullptr; }
+	void test_case_end(const doctest::CurrentTestCaseStats& in) override
+	{
+		if (!in.testCaseSuccess && Current)
+		{
+			FailedTests.push_back({Current->m_name, Current->m_file.c_str(), Current->m_line});
+		}
+		Current = nullptr;
+	}
 
 	void test_case_exception(const doctest::TestCaseException& /*in*/) override {}
 
@@ -57,7 +96,16 @@ struct TestListener : public doctest::IReporter
 
 	void test_case_skipped(const doctest::TestCaseData& /*in*/) override {}
 
-	const doctest::TestCaseData* Current = nullptr;
+	const doctest::TestCaseData*		  Current  = nullptr;
+	std::chrono::steady_clock::time_point RunStart = {};
+
+	struct FailedTestInfo
+	{
+		std::string Name;
+		std::string File;
+		unsigned	Line;
+	};
+	std::vector<FailedTestInfo> FailedTests;
 };
 
 struct TestRunner::Impl
-- 
cgit v1.2.3