zenserver process launch/termination improvements (#138)

* zenserver process launch/termination improvements
* fix GetPidStatus to return error code on Linux
* fix linux FindProcess()
* cleanup IsZombieProcess

1 files changed, 139 insertions, 69 deletions

diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 29d590d00..7f053747e 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -574,12 +574,12 @@ ZenServerInstance::~ZenServerInstance()
 }
 
 bool
-ZenServerInstance::SignalShutdown()
+ZenServerInstance::SignalShutdown(std::error_code& OutEc)
 {
 	if (m_ShutdownEvent)
 	{
-		m_ShutdownEvent->Set();
-		return true;
+		OutEc = m_ShutdownEvent->Set();
+		return !OutEc;
 	}
 	else
 	{
@@ -604,26 +604,53 @@ ZenServerInstance::Shutdown()
 			}
 			else
 			{
-				if (m_Process.IsRunning())
+				if (!m_Process.IsRunning())
+				{
+					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
+					int ExitCode = m_Process.GetExitCode();
+					m_Process.Reset();
+					return ExitCode;
+				}
+
+				ZEN_DEBUG("Requesting zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
+				std::error_code Ec;
+				if (SignalShutdown(Ec))
 				{
-					ZEN_DEBUG("Requesting zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
-					if (!SignalShutdown())
-					{
-						ZEN_INFO("Terminating zenserver process as we did not wait for it to get ready {}", m_Name);
-						int ExitCode = 111;
-						m_Process.Terminate(ExitCode);
-						ZEN_DEBUG("zenserver process {} ({}) terminated", m_Name, m_Process.Pid());
-						return ExitCode;
-					}
 					ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
 					while (!m_Process.Wait(1000))
 					{
+						if (!m_Process.IsValid())
+						{
+							ZEN_WARN("Wait abandoned by invalid process");
+							break;
+						}
+						if (!m_Process.IsRunning())
+						{
+							ZEN_WARN("Wait abandoned by exited process");
+							return 0;
+						}
 						ZEN_WARN("Waiting for zenserver process {} ({}) timed out", m_Name, m_Process.Pid());
 					}
+					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
+					int ExitCode = m_Process.GetExitCode();
+					m_Process.Reset();
+					return ExitCode;
 				}
-				ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
-				int ExitCode = m_Process.GetExitCode();
-				m_Process.Reset();
+				else if (Ec)
+				{
+					ZEN_WARN("Terminating zenserver proces as we failed to signal zenserver process {} ({}) to shut down. Reason: '{}'",
+							 m_Name,
+							 m_Process.Pid(),
+							 Ec.message());
+				}
+				else
+				{
+					ZEN_INFO("Terminating zenserver process as we did not wait for it to get ready {}", m_Name);
+				}
+
+				int ExitCode = 111;
+				m_Process.Terminate(ExitCode);
+				ZEN_DEBUG("zenserver process {} ({}) terminated", m_Name, m_Process.Pid());
 				return ExitCode;
 			}
 		}
@@ -644,11 +671,10 @@ ZenServerInstance::Shutdown()
 }
 
 void
-ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs)
+ZenServerInstance::SpawnServer(std::string_view ServerArgs, bool OpenConsole, int WaitTimeoutMs)
 {
 	ZEN_ASSERT(!m_Process.IsValid());  // Only spawn once
 
-	const int MyPid	  = zen::GetCurrentProcessId();
 	const int ChildId = ++ChildIdCounter;
 
 	ExtendableStringBuilder<32> ChildEventName;
@@ -664,50 +690,11 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 
 	const bool IsTest = m_Env.IsTestEnvironment();
 
-	if (IsTest)
-	{
-		if (!m_OwnerPid.has_value())
-		{
-			m_OwnerPid = MyPid;
-		}
-
-		CommandLine << " --test --log-id " << m_Name;
-		CommandLine << " --no-sentry";
-
-		if (AdditionalServerArgs.find("--system-dir") == std::string_view::npos)
-		{
-			CommandLine << " --system-dir ";
-			PathToUtf8((m_Env.CreateNewTestDir() / "system-dir").c_str(), CommandLine);
-		}
-	}
-
-	if (m_OwnerPid.has_value())
-	{
-		CommandLine << " --owner-pid " << m_OwnerPid.value();
-	}
-
 	CommandLine << " --child-id " << ChildEventName;
 
-	if (std::string_view ServerClass = m_Env.GetServerClass(); ServerClass.empty() == false)
-	{
-		CommandLine << " --http " << ServerClass;
-	}
-
-	if (BasePort)
+	if (!ServerArgs.empty())
 	{
-		CommandLine << " --port " << BasePort;
-		m_BasePort = gsl::narrow_cast<uint16_t>(BasePort);
-	}
-
-	if (!m_TestDir.empty())
-	{
-		CommandLine << " --data-dir ";
-		PathToUtf8(m_TestDir.c_str(), CommandLine);
-	}
-
-	if (!AdditionalServerArgs.empty())
-	{
-		CommandLine << " " << AdditionalServerArgs;
+		CommandLine << " " << ServerArgs;
 	}
 
 	std::filesystem::path CurrentDirectory = std::filesystem::current_path();
@@ -715,22 +702,31 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	ZEN_DEBUG("Spawning server '{}'", LogId);
 
 	uint32_t CreationFlags = 0;
-	if (!IsTest)
+	if (OpenConsole)
 	{
 		CreationFlags |= CreateProcOptions::Flag_NewConsole;
 	}
 
-	const std::filesystem::path BaseDir		  = m_Env.ProgramBaseDir();
-	const std::filesystem::path Executable	  = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
-	const std::filesystem::path OutputPath	  = std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log");
-	CreateProcOptions			CreateOptions = {.WorkingDirectory = &CurrentDirectory, .Flags = CreationFlags, .StdoutFile = OutputPath};
-	CreateProcResult			ChildPid	  = CreateProc(Executable, CommandLine.ToView(), CreateOptions);
+	const std::filesystem::path BaseDir	   = m_Env.ProgramBaseDir();
+	const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
+	const std::filesystem::path OutputPath =
+		OpenConsole ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log");
+	CreateProcOptions CreateOptions = {.WorkingDirectory = &CurrentDirectory, .Flags = CreationFlags, .StdoutFile = OutputPath};
+	CreateProcResult  ChildPid		= CreateProc(Executable, CommandLine.ToView(), CreateOptions);
 #if ZEN_PLATFORM_WINDOWS
-	if (!ChildPid && ::GetLastError() == ERROR_ELEVATION_REQUIRED)
+	if (!ChildPid)
 	{
-		ZEN_DEBUG("Regular spawn failed - spawning elevated server");
-		CreateOptions.Flags |= CreateProcOptions::Flag_Elevated;
-		ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions);
+		DWORD Error = GetLastError();
+		if (Error == ERROR_ELEVATION_REQUIRED)
+		{
+			ZEN_DEBUG("Regular spawn failed - spawning elevated server");
+			CreateOptions.Flags |= CreateProcOptions::Flag_Elevated;
+			ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions);
+		}
+		else
+		{
+			ThrowSystemError(Error, "Server spawn failed");
+		}
 	}
 #endif
 
@@ -764,6 +760,70 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 }
 
 void
+ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs)
+{
+	ZEN_ASSERT(!m_Process.IsValid());  // Only spawn once
+
+	const int MyPid	  = zen::GetCurrentProcessId();
+	const int ChildId = ++ChildIdCounter;
+
+	ExtendableStringBuilder<32> LogId;
+	LogId << "Zen" << ChildId;
+	m_Name = LogId.ToString();
+
+	ExtendableStringBuilder<512> CommandLine;
+	CommandLine << "zenserver" ZEN_EXE_SUFFIX_LITERAL;	// see CreateProc() call for actual binary path
+
+	const bool IsTest = m_Env.IsTestEnvironment();
+
+	if (IsTest)
+	{
+		if (!m_OwnerPid.has_value())
+		{
+			m_OwnerPid = MyPid;
+		}
+
+		CommandLine << " --test --log-id " << m_Name;
+		CommandLine << " --no-sentry";
+
+		if (AdditionalServerArgs.find("--system-dir") == std::string_view::npos)
+		{
+			CommandLine << " --system-dir ";
+			PathToUtf8((m_Env.CreateNewTestDir() / "system-dir").c_str(), CommandLine);
+		}
+	}
+
+	if (m_OwnerPid.has_value())
+	{
+		CommandLine << " --owner-pid " << m_OwnerPid.value();
+	}
+
+	if (std::string_view ServerClass = m_Env.GetServerClass(); ServerClass.empty() == false)
+	{
+		CommandLine << " --http " << ServerClass;
+	}
+
+	if (BasePort)
+	{
+		CommandLine << " --port " << BasePort;
+		m_BasePort = gsl::narrow_cast<uint16_t>(BasePort);
+	}
+
+	if (!m_TestDir.empty())
+	{
+		CommandLine << " --data-dir ";
+		PathToUtf8(m_TestDir.c_str(), CommandLine);
+	}
+
+	if (!AdditionalServerArgs.empty())
+	{
+		CommandLine << " " << AdditionalServerArgs;
+	}
+
+	SpawnServer(CommandLine, !IsTest, WaitTimeoutMs);
+}
+
+void
 ZenServerInstance::CreateShutdownEvent(int BasePort)
 {
 	ExtendableStringBuilder<32> ChildShutdownEventName;
@@ -877,6 +937,16 @@ ZenServerInstance::WaitUntilReady(int Timeout)
 	return true;
 }
 
+bool
+ZenServerInstance::WaitUntilExited(int Timeout, std::error_code& OutEc)
+{
+	if (m_Process.IsRunning())
+	{
+		return m_Process.Wait(Timeout, OutEc);
+	}
+	return false;
+}
+
 void
 ZenServerInstance::OnServerReady()
 {