kill stale test processes (zenserver, minio, nomad, consul) before and after CI test runs (#909)

Adds steps to the validate workflow on all platforms that kill any zenserver, minio, nomad, or consul processes launched from the build output directory. Runs before tests to clear stale processes from previous runs, and after tests (always, even on failure) to clean up.

2 files changed, 92 insertions, 4 deletions

diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index 601f000fe..99bf1e2d2 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -84,11 +84,39 @@ jobs:
       if: ${{ matrix.config == 'debug' }}
       run: if (Test-Path build/reports) { Remove-Item -Recurse -Force build/reports }
 
+    - name: Kill stale test processes
+      if: ${{ matrix.config == 'debug' }}
+      shell: powershell
+      run: |
+        $buildDir = "${{ github.workspace }}\build"
+        foreach ($name in @('zenserver', 'minio', 'nomad', 'consul')) {
+          $procs = Get-Process -Name $name -ErrorAction SilentlyContinue |
+            Where-Object { $_.Path -and $_.Path.StartsWith($buildDir, [System.StringComparison]::OrdinalIgnoreCase) }
+          foreach ($p in $procs) {
+            Write-Host "Killing stale $name (PID $($p.Id)): $($p.Path)"
+            $p | Stop-Process -Force -ErrorAction SilentlyContinue
+          }
+        }
+
     - name: Build & Test
       if: ${{ matrix.config == 'debug' }}
       run: |
         xmake test -v -y --junit
 
+    - name: Kill test processes
+      if: ${{ always() && matrix.config == 'debug' }}
+      shell: powershell
+      run: |
+        $buildDir = "${{ github.workspace }}\build"
+        foreach ($name in @('zenserver', 'minio', 'nomad', 'consul')) {
+          $procs = Get-Process -Name $name -ErrorAction SilentlyContinue |
+            Where-Object { $_.Path -and $_.Path.StartsWith($buildDir, [System.StringComparison]::OrdinalIgnoreCase) }
+          foreach ($p in $procs) {
+            Write-Host "Killing leftover $name (PID $($p.Id)): $($p.Path)"
+            $p | Stop-Process -Force -ErrorAction SilentlyContinue
+          }
+        }
+
     - name: Upload report
       if: ${{ (failure() || success()) && (matrix.config == 'debug') }}
       uses: actions/upload-artifact@v3-node20
@@ -152,11 +180,39 @@ jobs:
       shell: bash
       run: rm -rf build/reports
 
+    - name: Kill stale test processes
+      if: ${{ matrix.config == 'debug' }}
+      shell: bash
+      run: |
+        build_dir="${{ github.workspace }}/build"
+        for name in zenserver minio nomad consul; do
+          pgrep -a "$name" 2>/dev/null | while read -r pid cmd; do
+            if [[ "$cmd" == "$build_dir"* ]]; then
+              echo "Killing stale $name (PID $pid): $cmd"
+              kill -9 "$pid" 2>/dev/null || true
+            fi
+          done || true
+        done
+
     - name: Build & Test
       if: ${{ matrix.config == 'debug' }}
       run: |
         xmake test -v -y --junit
 
+    - name: Kill test processes
+      if: ${{ always() && matrix.config == 'debug' }}
+      shell: bash
+      run: |
+        build_dir="${{ github.workspace }}/build"
+        for name in zenserver minio nomad consul; do
+          pgrep -a "$name" 2>/dev/null | while read -r pid cmd; do
+            if [[ "$cmd" == "$build_dir"* ]]; then
+              echo "Killing leftover $name (PID $pid): $cmd"
+              kill -9 "$pid" 2>/dev/null || true
+            fi
+          done || true
+        done
+
     - name: Upload report
       if: ${{ (failure() || success()) && (matrix.config == 'debug') }}
       uses: actions/upload-artifact@v3-node20
@@ -221,11 +277,39 @@ jobs:
       shell: bash
       run: rm -rf build/reports
 
+    - name: Kill stale test processes
+      if: ${{ matrix.config == 'debug' }}
+      shell: bash
+      run: |
+        build_dir="${{ github.workspace }}/build"
+        for name in zenserver minio nomad consul; do
+          pgrep -a "$name" 2>/dev/null | while read -r pid cmd; do
+            if [[ "$cmd" == "$build_dir"* ]]; then
+              echo "Killing stale $name (PID $pid): $cmd"
+              kill -9 "$pid" 2>/dev/null || true
+            fi
+          done || true
+        done
+
     - name: Build & Test
       if: ${{ matrix.config == 'debug' }}
       run: |
         xmake test -v -y --junit
 
+    - name: Kill test processes
+      if: ${{ always() && matrix.config == 'debug' }}
+      shell: bash
+      run: |
+        build_dir="${{ github.workspace }}/build"
+        for name in zenserver minio nomad consul; do
+          pgrep -a "$name" 2>/dev/null | while read -r pid cmd; do
+            if [[ "$cmd" == "$build_dir"* ]]; then
+              echo "Killing leftover $name (PID $pid): $cmd"
+              kill -9 "$pid" 2>/dev/null || true
+            fi
+          done || true
+        done
+
     - name: Upload report
       if: ${{ (failure() || success()) && (matrix.config == 'debug') }}
       uses: actions/upload-artifact@v3-node20
diff --git a/src/zencompute/computeservice.cpp b/src/zencompute/computeservice.cpp
index 58761556a..aaf34cbe2 100644
--- a/src/zencompute/computeservice.cpp
+++ b/src/zencompute/computeservice.cpp
@@ -1447,15 +1447,19 @@ ComputeServiceSession::Impl::NotifyQueueActionComplete(int QueueId, int Lsn, Run
 		return;
 	}
 
+	bool WasActive = false;
 	Queue->m_Lock.WithExclusiveLock([&] {
-		Queue->ActiveLsns.erase(Lsn);
+		WasActive = Queue->ActiveLsns.erase(Lsn) > 0;
 		Queue->FinishedLsns.insert(Lsn);
 	});
 
-	const int PreviousActive = Queue->ActiveCount.fetch_sub(1, std::memory_order_relaxed);
-	if (PreviousActive == 1)
+	if (WasActive)
 	{
-		Queue->IdleSince.store(GetHifreqTimerValue(), std::memory_order_relaxed);
+		const int PreviousActive = Queue->ActiveCount.fetch_sub(1, std::memory_order_relaxed);
+		if (PreviousActive == 1)
+		{
+			Queue->IdleSince.store(GetHifreqTimerValue(), std::memory_order_relaxed);
+		}
 	}
 
 	switch (ActionState)