pause, resume and abort running builds cmd (#421)

- Feature: `zen builds pause`, `zen builds resume` and `zen builds abort` commands to control a running `zen builds` command
  - `--process-id` the process id to control, if omitted it tries to find a running process using the same executable as itself
- Improvement: Process report now indicates if it is pausing or aborting

2 files changed, 10 insertions, 6 deletions

diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index f7e63433b..9f2e826d6 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -1588,7 +1588,8 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext&  Co
 	Stopwatch		 Timer;
 	auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); });
 	std::atomic<bool> AbortFlag;
-	ParallelWork	  Work(AbortFlag);
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
 	ZEN_INFO("Replaying {} requests", RequestCount);
 	for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex)
 	{
@@ -1638,8 +1639,8 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext&  Co
 			}
 		});
 	}
-	Work.Wait(10000, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-		ZEN_UNUSED(IsAborted);
+	Work.Wait(10000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+		ZEN_UNUSED(IsAborted, IsPaused);
 		ZEN_INFO("Replayed {} of {} requests, elapsed {}",
 				 RequestCount - PendingWork,
 				 RequestCount,
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 3ec4373a2..a2e73380f 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1597,7 +1597,8 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo
 	};
 
 	std::atomic<bool> AbortFlag;
-	ParallelWork	  Work(AbortFlag);
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
 	for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++)
 	{
 		if (OptionalWorkerPool)
@@ -2111,7 +2112,8 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid>																	  ChunkIds,
 	{
 		std::atomic_bool  Result = true;
 		std::atomic<bool> AbortFlag;
-		ParallelWork	  Work(AbortFlag);
+		std::atomic<bool> PauseFlag;
+		ParallelWork	  Work(AbortFlag, PauseFlag);
 
 		for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++)
 		{
@@ -3890,7 +3892,8 @@ ProjectStore::Flush()
 
 	WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst);
 	std::atomic<bool> AbortFlag;
-	ParallelWork	  Work(AbortFlag);
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
 
 	for (const Ref<Project>& Project : Projects)
 	{