pause, resume and abort running builds cmd (#421)

- Feature: `zen builds pause`, `zen builds resume` and `zen builds abort` commands to control a running `zen builds` command
  - `--process-id` the process id to control, if omitted it tries to find a running process using the same executable as itself
- Improvement: Process report now indicates if it is pausing or aborting

5 files changed, 44 insertions, 31 deletions

diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp
index c7532e098..cd1bf7dd7 100644
--- a/src/zenutil/chunkedcontent.cpp
+++ b/src/zenutil/chunkedcontent.cpp
@@ -758,14 +758,15 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span
 }
 
 ChunkedFolderContent
-ChunkFolderContent(ChunkingStatistics&												 Stats,
-				   WorkerThreadPool&												 WorkerPool,
-				   const std::filesystem::path&										 RootPath,
-				   const FolderContent&												 Content,
-				   const ChunkingController&										 InChunkingController,
-				   int32_t															 UpdateIntervalMS,
-				   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback,
-				   std::atomic<bool>&												 AbortFlag)
+ChunkFolderContent(ChunkingStatistics&																Stats,
+				   WorkerThreadPool&																WorkerPool,
+				   const std::filesystem::path&														RootPath,
+				   const FolderContent&																Content,
+				   const ChunkingController&														InChunkingController,
+				   int32_t																			UpdateIntervalMS,
+				   std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback,
+				   std::atomic<bool>&																AbortFlag,
+				   std::atomic<bool>&																PauseFlag)
 {
 	ZEN_TRACE_CPU("ChunkFolderContent");
 
@@ -804,7 +805,7 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 
 		RwLock Lock;
 
-		ParallelWork Work(AbortFlag);
+		ParallelWork Work(AbortFlag, PauseFlag);
 
 		for (uint32_t PathIndex : Order)
 		{
@@ -831,10 +832,9 @@ ChunkFolderContent(ChunkingStatistics&												 Stats,
 							  });
 		}
 
-		Work.Wait(UpdateIntervalMS, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
-			ZEN_UNUSED(IsAborted);
+		Work.Wait(UpdateIntervalMS, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
 			ZEN_UNUSED(PendingWork);
-			UpdateCallback(Work.IsAborted(), Work.PendingWork().Remaining());
+			UpdateCallback(IsAborted, IsPaused, Work.PendingWork().Remaining());
 		});
 	}
 	return Result;
diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h
index 225b1a3a5..306a5d990 100644
--- a/src/zenutil/include/zenutil/chunkedcontent.h
+++ b/src/zenutil/include/zenutil/chunkedcontent.h
@@ -132,14 +132,15 @@ struct ChunkingStatistics
 	uint64_t			  ElapsedWallTimeUS	   = 0;
 };
 
-ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&												  Stats,
-										WorkerThreadPool&												  WorkerPool,
-										const std::filesystem::path&									  RootPath,
-										const FolderContent&											  Content,
-										const ChunkingController&										  InChunkingController,
-										int32_t															  UpdateIntervalMS,
-										std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&& UpdateCallback,
-										std::atomic<bool>&												  AbortFlag);
+ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&			 Stats,
+										WorkerThreadPool&			 WorkerPool,
+										const std::filesystem::path& RootPath,
+										const FolderContent&		 Content,
+										const ChunkingController&	 InChunkingController,
+										int32_t						 UpdateIntervalMS,
+										std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback,
+										std::atomic<bool>&																 AbortFlag,
+										std::atomic<bool>&																 PauseFlag);
 
 ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content);
 
diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h
index 08e730b28..d7e986551 100644
--- a/src/zenutil/include/zenutil/parallelwork.h
+++ b/src/zenutil/include/zenutil/parallelwork.h
@@ -12,13 +12,13 @@ namespace zen {
 class ParallelWork
 {
 public:
-	ParallelWork(std::atomic<bool>& AbortFlag);
+	ParallelWork(std::atomic<bool>& AbortFlag, std::atomic<bool>& PauseFlag);
 
 	~ParallelWork();
 
-	typedef std::function<void(std::atomic<bool>& AbortFlag)>						 WorkCallback;
-	typedef std::function<void(std::exception_ptr Ex, std::atomic<bool>& AbortFlag)> ExceptionCallback;
-	typedef std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>			 UpdateCallback;
+	typedef std::function<void(std::atomic<bool>& AbortFlag)>							   WorkCallback;
+	typedef std::function<void(std::exception_ptr Ex, std::atomic<bool>& AbortFlag)>	   ExceptionCallback;
+	typedef std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)> UpdateCallback;
 
 	void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError = {})
 	{
@@ -28,6 +28,10 @@ public:
 			WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] {
 				try
 				{
+					while (m_PauseFlag && !m_AbortFlag)
+					{
+						Sleep(2000);
+					}
 					Work(m_AbortFlag);
 				}
 				catch (...)
@@ -59,6 +63,7 @@ private:
 	void			  RethrowErrors();
 
 	std::atomic<bool>& m_AbortFlag;
+	std::atomic<bool>& m_PauseFlag;
 	bool			   m_DispatchComplete = false;
 	Latch			   m_PendingWork;
 
diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp
index ecacc4b5a..67fc03c04 100644
--- a/src/zenutil/parallelwork.cpp
+++ b/src/zenutil/parallelwork.cpp
@@ -14,7 +14,10 @@
 
 namespace zen {
 
-ParallelWork::ParallelWork(std::atomic<bool>& AbortFlag) : m_AbortFlag(AbortFlag), m_PendingWork(1)
+ParallelWork::ParallelWork(std::atomic<bool>& AbortFlag, std::atomic<bool>& PauseFlag)
+: m_AbortFlag(AbortFlag)
+, m_PauseFlag(PauseFlag)
+, m_PendingWork(1)
 {
 }
 
@@ -59,7 +62,7 @@ ParallelWork::Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback)
 
 	while (!m_PendingWork.Wait(UpdateIntervalMS))
 	{
-		UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining());
+		UpdateCallback(m_AbortFlag.load(), m_PauseFlag.load(), m_PendingWork.Remaining());
 	}
 
 	RethrowErrors();
@@ -111,7 +114,8 @@ ParallelWork::RethrowErrors()
 TEST_CASE("parallellwork.nowork")
 {
 	std::atomic<bool> AbortFlag;
-	ParallelWork	  Work(AbortFlag);
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
 	Work.Wait();
 }
 
@@ -120,7 +124,8 @@ TEST_CASE("parallellwork.basic")
 	WorkerThreadPool WorkerPool(2);
 
 	std::atomic<bool> AbortFlag;
-	ParallelWork	  Work(AbortFlag);
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
 	for (uint32_t I = 0; I < 5; I++)
 	{
 		Work.ScheduleWork(WorkerPool, [](std::atomic<bool>& AbortFlag) { CHECK(!AbortFlag); });
@@ -133,7 +138,8 @@ TEST_CASE("parallellwork.throws_in_work")
 	WorkerThreadPool WorkerPool(2);
 
 	std::atomic<bool> AbortFlag;
-	ParallelWork	  Work(AbortFlag);
+	std::atomic<bool> PauseFlag;
+	ParallelWork	  Work(AbortFlag, PauseFlag);
 	for (uint32_t I = 0; I < 10; I++)
 	{
 		Work.ScheduleWork(WorkerPool, [I](std::atomic<bool>& AbortFlag) {
@@ -158,7 +164,8 @@ TEST_CASE("parallellwork.throws_in_dispatch")
 	try
 	{
 		std::atomic<bool> AbortFlag;
-		ParallelWork	  Work(AbortFlag);
+		std::atomic<bool> PauseFlag;
+		ParallelWork	  Work(AbortFlag, PauseFlag);
 		for (uint32_t I = 0; I < 5; I++)
 		{
 			Work.ScheduleWork(WorkerPool, [I, &ExecutedCount](std::atomic<bool>& AbortFlag) {
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index bfa0d3c49..a5b342cb0 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -1059,7 +1059,7 @@ ZenServerInstance::Terminate()
 	const std::filesystem::path BaseDir	   = m_Env.ProgramBaseDir();
 	const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
 	ProcessHandle				RunningProcess;
-	std::error_code				Ec = FindProcess(Executable, RunningProcess);
+	std::error_code				Ec = FindProcess(Executable, RunningProcess, /*IncludeSelf*/ false);
 	if (Ec)
 	{
 		throw std::system_error(Ec, fmt::format("failed to look up running server executable '{}'", Executable));