junit test reporting (#239)

- Feature: `--junit` switch to `xmake test` to generate junit style reports of tests.
- Feature: CI build on GitHub now uploads junit test reports as artifact to the check for PR validation and mainline validation

3 files changed, 75 insertions, 33 deletions

diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index a12878e87..e410a831c 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -73,13 +73,6 @@ jobs:
         .\vcpkg.exe integrate install
         cd ..
 
-#    - name: Cache vcpkg
-#      uses: actions/cache@v2
-#      with:
-#        path: |
-#          ${{ github.workspace }}\.vcpkg\installed
-#        key: ${{ runner.os }}-validate-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v1
-
     - name: Config
       run: |
         xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }}
@@ -89,10 +82,17 @@ jobs:
     - name: Build & Test
       if: ${{ matrix.config == 'debug' }}
       run: |
-        xmake test -v -y
+        xmake test -v -y --junit
       env:
         VCPKG_ROOT: ${{ github.workspace }}/.vcpkg
 
+    - name: Upload report
+      if: ${{ (failure() || success()) && (matrix.config == 'debug') }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: reports-win64
+        path: build/reports/*.xml
+
     - name: Bundle
       if: ${{ matrix.config == 'release' }}
       run: |
@@ -144,13 +144,6 @@ jobs:
         ./bootstrap-vcpkg.sh
         cd ..
 
-#    - name: Cache vcpkg
-#      uses: actions/cache@v2
-#      with:
-#        path: |
-#          ${{ github.workspace }}/.vcpkg/installed
-#        key: ${{ runner.os }}-validate-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v1
-
     - name: Config
       run: |
         xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }}
@@ -160,10 +153,17 @@ jobs:
     - name: Build & Test
       if: ${{ matrix.config == 'debug' }}
       run: |
-        xmake test -v -y
+        xmake test -v -y --junit
       env:
         VCPKG_ROOT: ${{ github.workspace }}/.vcpkg
 
+    - name: Upload report
+      if: ${{ (failure() || success()) && (matrix.config == 'debug') }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: reports-linux
+        path: build/reports/*.xml
+
     - name: Bundle
       if: ${{ matrix.config == 'release' }}
       run: |
@@ -209,13 +209,6 @@ jobs:
         ./bootstrap-vcpkg.sh
         cd ..
 
-#    - name: Cache vcpkg
-#      uses: actions/cache@v2
-#      with:
-#        path: |
-#          ${{ github.workspace }}/.vcpkg/installed
-#        key: ${{ runner.os }}-validate-${{ matrix.config }}-${{env.VCPKG_VERSION}}-${{ hashFiles('xmake.lua') }}-${{ matrix.arch }}-v1
-
     - name: Config
       run: |
         xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }}
@@ -225,10 +218,17 @@ jobs:
     - name: Build & Test
       if: ${{ matrix.config == 'debug' }}
       run: |
-        xmake test -v -y
+        xmake test -v -y --junit
       env:
         VCPKG_ROOT: ${{ github.workspace }}/.vcpkg
 
+    - name: Upload report
+      if: ${{ (failure() || success()) && (matrix.config == 'debug') }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: reports-macos
+        path: build/reports/*.xml
+
     - name: Bundle
       if: ${{ matrix.config == 'release' }}
       run: |
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ec2b541b3..8183bf774 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,8 @@
   - `--stride` The stride to use when selecting requests to playback
   - `--onhost` Replay the recording inside the zenserver bypassing http overhead
   - `--showmethodstats` Show statistics of which RPC methods are used
+- Feature: `--junit` switch to `xmake test` to generate junit style reports of tests.
+- Feature: CI build on GitHub now uploads junit test reports as artifact to the check for PR validation and mainline validation
 - Bugfix: Make sure async responses are sent async correctly in httpsys
 - Improvement: FileCas now keeps an up to date index of all the entries improving performance when getting cache misses on large payloads
 - Changed: Exit with failure code on port conflict rather than reporting crash to Sentry
diff --git a/xmake.lua b/xmake.lua
index e93b6f464..dbbfeb9c2 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -185,12 +185,17 @@ task("test")
         usage = "xmake runtest [core|store|server|integration|all]",
         description = "Run Zen tests",
         options = {
-            {'r', "run", "kv", "all", "Run test(s)", " - all", " - core ", " - store", " - server", " - integration"}
+            {'r', "run", "kv", "all", "Run test(s)", " - all", " - core ", " - store", " - server", " - integration"},
+            {'j', "junit", "k", nil, "Enable junit report output"}
         }
     }
     on_run(function()
         import("core.base.option")
+        import("core.project.config")
+        import("core.project.project")
         
+        config.load()
+
         local testname = option.get("run")
         local available_tests = {
             core = "zencore-test",
@@ -215,13 +220,48 @@ task("test")
                 tests[name] = test
             end
         end
-        
-        for name, test in pairs(tests) do
-            printf("=== %s ===\n", test)
-            local cmd = string.format("xmake run %s", test)
-            if name == "server" then
-                cmd = string.format("xmake run %s test", test)
-            end
-            print(os.exec(cmd))
+
+        local use_junit_reporting = option.get("junit")
+        local junit_report_files = {}
+
+        local junit_report_dir
+        if use_junit_reporting then
+            junit_report_dir = path.join(os.projectdir(), config.get("buildir"), "reports")
+            os.mkdir(junit_report_dir)
         end
+
+        try
+        {
+            function()
+                for name, test in pairs(tests) do
+                    printf("=== %s ===\n", test)
+                    local cmd = string.format("xmake run %s", test)
+                    if name == "server" then
+                        cmd = string.format("xmake run %s test", test)
+                    end
+                    if use_junit_reporting then
+                        local target = project.target(test)
+                        local junit_report_file = path.join(junit_report_dir, string.format("junit-%s-%s-%s.xml", config.plat(), arch, test))
+                        junit_report_files[test] = junit_report_file
+                        cmd = string.format("%s --reporters=junit --out=%s", cmd, junit_report_file)
+                    end
+
+                    os.exec(cmd)
+                end
+            end,
+
+            finally
+            {
+                function (ok, errors)
+                    for test, junit_report_file in pairs(junit_report_files) do
+                        printf("=== report - %s ===\n", test)
+                        local data = io.readfile(junit_report_file)
+                        print(data)
+                    end
+                    if (errors) then
+                        raise(errors)
+                    end
+                end
+            }
+        }
     end)