Add test suites (#799)

Makes all test cases part of a test suite. Test suites are named after the module and the name of the file containing the implementation of the test.

* This allows for better and more predictable filtering of which test cases to run which should also be able to reduce the time CI spends in tests since it can filter on the tests for that particular module.

Also improves `xmake test` behaviour:

* instead of an explicit list of projects just enumerate the test projects which are available based on build system state
* also introduces logic to avoid running `xmake config` unnecessarily which would invalidate the existing build and do lots of unnecessary work since dependencies were invalidated by the updated config
* also invokes build only for the chosen test targets

As a bonus, also adds `xmake sln --open` which allows opening IDE after generation of solution/xmake project is done.

1 files changed, 84 insertions, 29 deletions

diff --git a/xmake.lua b/xmake.lua
index 10f5c9c3a..0d8f53a16 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -329,29 +329,46 @@ task("precommit")
 
 task("sln")
     set_menu {
-        usage = "xmake sln",
+        usage = "xmake sln [--open]",
         description = "Generate IDE project files",
+        options = {
+            {'o', "open", "k", nil, "Open the generated project in the IDE after generation"},
+        }
     }
     if is_os("windows") then
         on_run(function ()
-                print(os.exec("xmake project --yes --kind=vsxmake2022 -m release,debug -a x64"))
+            import("core.base.option")
+            os.exec("xmake project --yes --kind=vsxmake2022 -m release,debug -a x64")
+            if option.get("open") then
+                local sln = path.join(os.projectdir(), "vsxmake2022", path.filename(os.projectdir()) .. ".sln")
+                printf("opening %s\n", sln)
+                try { function() os.execv("explorer", {sln}) end, catch { function() end } }
+            end
         end)
     elseif is_os("macosx") then
         on_run(function ()
-                print(os.exec("xmake project --yes --kind=xcode -m release,debug -a x64,arm64"))
+            import("core.base.option")
+            os.exec("xmake project --yes --kind=xcode -m release,debug -a x64,arm64")
+            if option.get("open") then
+                local xcproj = path.join(os.projectdir(), path.filename(os.projectdir()) .. ".xcodeproj")
+                printf("opening %s\n", xcproj)
+                os.exec("open \"%s\"", xcproj)
+            end
         end)
     end
 
 task("test")
     set_menu {
-        usage = "xmake test --run=[core|store|http|server|integration|util|remotestore|all] (comma-separated)",
+        usage = "xmake test --run=[name|all] [-- extra-args...] (use --list to see available tests)",
         description = "Run Zen tests",
         options = {
-            {'r', "run", "kv", "all", "Run test(s) - comma-separated", " - all", " - core", " - http", " - util", " - store", " - remotestore", " - server", " - integration"},
+            {'r', "run", "kv", "all", "Run test(s) - comma-separated"},
+            {'l', "list", "k", nil, "List available test names"},
             {'j', "junit", "k", nil, "Enable junit report output"},
             {'n', "noskip", "k", nil, "Run skipped tests (passes --no-skip to doctest)"},
             {nil, "repeat", "kv", nil, "Repeat tests N times (stops on first failure)"},
-            {'v', "verbose", "k", nil, "Route child process output to stdout (zenserver-test)"}
+            {'v', "verbose", "k", nil, "Route child process output to stdout (zenserver-test)"},
+            {nil, "arguments", "vs", nil, "Extra arguments passed to test runners (after --)"}
         }
     }
     on_run(function()
@@ -361,33 +378,41 @@ task("test")
 
         config.load()
 
-        local testname = option.get("run")
-
-        -- Ordered list of available tests (order defines execution order)
-        local available_tests = {
-            {"core",        "zencore-test"},
-            {"http",        "zenhttp-test"},
-            {"util",        "zenutil-test"},
-            {"store",       "zenstore-test"},
-            {"remotestore", "zenremotestore-test"},
-            {"server",      "zenserver"},
-            {"integration", "zenserver-test"},
+        -- Override table: target name -> short name (for targets that don't follow convention)
+        local short_name_overrides = {
+            ["zenserver-test"] = "integration",
         }
 
-        local plat, arch
-        if is_host("windows") then
-            plat = "windows"
-            arch = "x64"
-        elseif is_host("macosx") then
-            plat = "macosx"
-            arch = is_arch("arm64") and "arm64" or "x86_64"
-        else
-            plat = "linux"
-            arch = "x86_64"
+        -- Build test list from targets in the "tests" group
+        local available_tests = {}
+        for name, target in pairs(project.targets()) do
+            if target:get("group") == "tests" and name:endswith("-test") then
+                local short = short_name_overrides[name]
+                if not short then
+                    -- Derive short name: "zencore-test" -> "core"
+                    short = name
+                    if short:startswith("zen") then short = short:sub(4) end
+                    if short:endswith("-test") then short = short:sub(1, -6) end
+                end
+                table.insert(available_tests, {short, name})
+            end
+        end
+
+        -- Add non-test-group entries that have a test subcommand
+        table.insert(available_tests, {"server", "zenserver"})
+
+        table.sort(available_tests, function(a, b) return a[1] < b[1] end)
+
+        -- Handle --list: print discovered test names and exit
+        if option.get("list") then
+            printf("Available tests:\n")
+            for _, entry in ipairs(available_tests) do
+                printf("  %-16s -> %s\n", entry[1], entry[2])
+            end
+            return
         end
 
-        print(os.exec("xmake config -c -m debug -p "..plat.." -a "..arch))
-        print(os.exec("xmake"))
+        local testname = option.get("run")
 
         -- Parse comma-separated test names into a set
         local requested = {}
@@ -420,10 +445,37 @@ task("test")
             raise("no tests match specification: '%s'", testname)
         end
 
+        local plat, arch
+        if is_host("windows") then
+            plat = "windows"
+            arch = "x64"
+        elseif is_host("macosx") then
+            plat = "macosx"
+            arch = is_arch("arm64") and "arm64" or "x86_64"
+        else
+            plat = "linux"
+            arch = "x86_64"
+        end
+
+        -- Only reconfigure if current config doesn't already match
+        if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch then
+            os.exec("xmake config -c -m debug -p %s -a %s", plat, arch)
+        end
+
+        -- Build targets we're going to run
+        if requested["all"] then
+            os.exec("xmake build -y")
+        else
+            for _, entry in ipairs(tests) do
+                os.exec("xmake build -y %s", entry.target)
+            end
+        end
+
         local use_junit_reporting = option.get("junit")
         local use_noskip = option.get("noskip")
         local use_verbose = option.get("verbose")
         local repeat_count = tonumber(option.get("repeat")) or 1
+        local extra_args = option.get("arguments") or {}
         local junit_report_files = {}
 
         local junit_report_dir
@@ -540,6 +592,9 @@ task("test")
                 if use_verbose and name == "integration" then
                     cmd = string.format("%s --verbose", cmd)
                 end
+                for _, arg in ipairs(extra_args) do
+                    cmd = string.format("%s %s", cmd, arg)
+                end
 
                 -- Tell TestListener where to write the summary
                 local summary_file = path.join(summary_dir, target .. ".txt")