fix race contdition when signaling shutdown of process and waiting for completion (#539)

4 files changed, 63 insertions, 13 deletions

diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp
index 1f1b1b8f5..99d7cdc61 100644
--- a/src/zencore/thread.cpp
+++ b/src/zencore/thread.cpp
@@ -606,7 +606,7 @@ ProcessHandle::Terminate(int ExitCode)
 	bSuccess		 = (WaitResult != WAIT_OBJECT_0);
 #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 	ZEN_UNUSED(ExitCode);
-	bSuccess			= (kill(m_Pid, SIGKILL) == 0);
+	bSuccess = (kill(m_Pid, SIGKILL) == 0);
 #endif
 
 	if (!bSuccess)
@@ -650,25 +650,65 @@ ProcessHandle::Wait(int TimeoutMs)
 			break;
 	}
 #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	if (TimeoutMs < 0)
+	{
+		int WaitState = 0;
+		int Res		  = waitpid(m_Pid, &WaitState, WCONTINUED | WUNTRACED);
+		if (Res == -1)
+		{
+			int32_t LastError = zen::GetLastError();
+			if (LastError == ECHILD || LastError == ESRCH)
+			{
+				return true;
+			}
+			ThrowSystemError(static_cast<uint32_t>(LastError), "Process::Wait waitpid failed"sv);
+		}
+		if (WIFEXITED(WaitState))
+		{
+			return true;
+		}
+		if (WIFSIGNALED(WaitState))
+		{
+			return true;
+		}
+		return false;
+	}
+
 	const int SleepMs	= 20;
 	timespec  SleepTime = {0, SleepMs * 1000 * 1000};
-	for (int i = 0;; i += SleepMs)
+	for (int SleepedTimeMS = 0;; SleepedTimeMS += SleepMs)
 	{
-#	if ZEN_PLATFORM_MAC
 		int WaitState = 0;
-		waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED);
-#	endif
+		int Res		  = waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED);
+		if (Res == -1)
+		{
+			int32_t LastError = zen::GetLastError();
+			if (LastError == ECHILD || LastError == ESRCH)
+			{
+				return true;
+			}
+			ThrowSystemError(static_cast<uint32_t>(LastError), "Process::Wait waitpid failed"sv);
+		}
+		if (WIFEXITED(WaitState))
+		{
+			return true;
+		}
+		if (WIFSIGNALED(WaitState))
+		{
+			return true;
+		}
 
 		if (kill(m_Pid, 0) < 0)
 		{
-			if (zen::GetLastError() == ESRCH)
+			int32_t LastError = zen::GetLastError();
+			if (LastError == ECHILD || LastError == ESRCH)
 			{
 				return true;
 			}
-			break;
+			ThrowSystemError(static_cast<uint32_t>(LastError), "Process::Wait kill failed"sv);
 		}
 
-		if (TimeoutMs >= 0 && i >= TimeoutMs)
+		if (SleepedTimeMS >= TimeoutMs)
 		{
 			return false;
 		}
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 69cc2bbf5..44330d4c8 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -198,13 +198,15 @@ ZenEntryPoint::Run()
 		ShutdownThread.reset(new std::thread{[&] {
 			SetCurrentThreadName("shutdown_monitor");
 
-			ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}'", ShutdownEventName);
+			ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}",
+					 ShutdownEventName,
+					 zen::GetCurrentProcessId());
 
 			if (ShutdownEvent->Wait())
 			{
 				if (!IsApplicationExitRequested())
 				{
-					ZEN_INFO("shutdown signal received");
+					ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId());
 					Server.RequestExit(0);
 				}
 			}
@@ -244,7 +246,7 @@ ZenEntryPoint::Run()
 	}
 	catch (std::exception& e)
 	{
-		ZEN_CRITICAL("Caught exception in main: {}", e.what());
+		ZEN_CRITICAL("Caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what());
 		if (!IsApplicationExitRequested())
 		{
 			RequestApplicationExit(1);
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 7111900ec..9d3502a23 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -720,7 +720,7 @@ ZenServer::CheckSigInt()
 {
 	if (utils::SignalCounter[SIGINT] > 0)
 	{
-		ZEN_INFO("SIGINT triggered (Ctrl+C), exiting");
+		ZEN_INFO("SIGINT triggered (Ctrl+C) for process {}, exiting", zen::GetCurrentProcessId());
 		RequestExit(128 + SIGINT);
 		return;
 	}
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 83c6668ba..00736321c 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -468,7 +468,14 @@ ZenServerInstance::ZenServerInstance(ZenServerEnvironment& TestEnvironment) : m_
 
 ZenServerInstance::~ZenServerInstance()
 {
-	Shutdown();
+	try
+	{
+		Shutdown();
+	}
+	catch (const std::exception& Err)
+	{
+		ZEN_ERROR("Shutting down zenserver instance failed, reason: '{}'", Err.what());
+	}
 }
 
 void
@@ -491,6 +498,7 @@ ZenServerInstance::Shutdown()
 		else
 		{
 			SignalShutdown();
+			ZEN_DEBUG("Waiting for zenserver process {} to shut down", m_Process.Pid());
 			m_Process.Wait();
 			m_Process.Reset();
 		}