zenserver process launch/termination improvements (#138)

* zenserver process launch/termination improvements
* fix GetPidStatus to return error code on Linux
* fix linux FindProcess()
* cleanup IsZombieProcess

8 files changed, 322 insertions, 131 deletions

diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp
index 5344a078d..c65dc5d5a 100644
--- a/src/zen/cmds/up_cmd.cpp
+++ b/src/zen/cmds/up_cmd.cpp
@@ -17,13 +17,15 @@ namespace zen {
 UpCommand::UpCommand()
 {
 	m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("0"), "<hostport>");
-	m_Options.add_option("lifetime",
-						 "",
-						 "owner-pid",
-						 "Specify owning process id",
-						 cxxopts::value(m_OwnerPid)->default_value("0"),
-						 "<identifier>");
 	m_Options.add_option("", "b", "base-dir", "Parent folder of server executable", cxxopts::value(m_ProgramBaseDir), "<directory>");
+	m_Options
+		.add_option("", "c", "show-console", "Open a console window for the zenserver process", cxxopts::value(m_ShowConsole), "<console>");
+	m_Options.add_option("",
+						 "l",
+						 "show-log",
+						 "Show the output log of the zenserver process after successful start",
+						 cxxopts::value(m_ShowLog),
+						 "<showlog>");
 }
 
 UpCommand::~UpCommand() = default;
@@ -31,6 +33,8 @@ UpCommand::~UpCommand() = default;
 int
 UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 {
+	using namespace std::literals;
+
 	ZEN_UNUSED(GlobalOptions);
 
 	if (!ParseOptions(argc, argv))
@@ -38,6 +42,11 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		return 0;
 	}
 
+	if (m_ShowConsole && m_ShowLog)
+	{
+		throw OptionParseException("--show-console can not be used in combination with --show-log");
+	}
+
 	{
 		ZenServerState State;
 		if (State.InitializeReadOnly())
@@ -76,11 +85,12 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	ZenServerEnvironment ServerEnvironment;
 	ServerEnvironment.Initialize(m_ProgramBaseDir);
 	ZenServerInstance Server(ServerEnvironment);
-	if (m_OwnerPid != 0)
+	std::string		  ServerArguments = GlobalOptions.PassthroughCommandLine;
+	if ((m_Port != 0) && (ServerArguments.find("--port"sv) == std::string::npos))
 	{
-		Server.SetOwnerPid(m_OwnerPid);
+		ServerArguments.append(fmt::format(" --port {}", m_Port));
 	}
-	Server.SpawnServer(m_Port, GlobalOptions.PassthroughCommandLine);
+	Server.SpawnServer(ServerArguments, m_ShowConsole, /*WaitTimeoutMs*/ 0);
 
 	int Timeout = 10000;
 
@@ -90,15 +100,29 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		{
 			ZEN_ERROR("zen server launch failed (timed out), terminating");
 			Server.Terminate();
+			if (!m_ShowConsole)
+			{
+				ZEN_CONSOLE("{}", Server.GetLogOutput());
+			}
 			return 111;
 		}
 		int ReturnCode = Server.Shutdown();
-		ZEN_CONSOLE("{}", Server.GetLogOutput());
+		if (!m_ShowConsole)
+		{
+			ZEN_CONSOLE("{}", Server.GetLogOutput());
+		}
 		return ReturnCode;
 	}
 	else
 	{
-		ZEN_CONSOLE("zen server up");
+		if (m_ShowLog)
+		{
+			ZEN_CONSOLE("{}", Server.GetLogOutput());
+		}
+		else
+		{
+			ZEN_CONSOLE("zen server up");
+		}
 	}
 	return 0;
 }
@@ -225,11 +249,30 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			ZEN_CONSOLE("attached to server on port {} (pid {}), requesting shutdown", EntryPort, ServerProcessPid);
 
-			Server.Shutdown();
-
-			ZEN_CONSOLE("shutdown complete");
-
-			return 0;
+			std::error_code Ec;
+			if (Server.SignalShutdown(Ec) && !Ec)
+			{
+				Stopwatch Timer;
+				while (Timer.GetElapsedTimeMs() < 10000)
+				{
+					if (Server.WaitUntilExited(100, Ec) && !Ec)
+					{
+						ZEN_CONSOLE("shutdown complete");
+						return 0;
+					}
+					else if (Ec)
+					{
+						ZEN_CONSOLE("Waiting for server on port {} (pid {}) failed. Reason: '{}'",
+									EntryPort,
+									ServerProcessPid,
+									Ec.message());
+					}
+				}
+			}
+			else if (Ec)
+			{
+				ZEN_CONSOLE("requesting shutdown of server on port {} failed. Reason: '{}'", EntryPort, Ec.message());
+			}
 		}
 		catch (const std::exception& Ex)
 		{
@@ -240,11 +283,11 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		// handle to determine when the server has shut down. Thus we signal that we would like
 		// a shutdown via the shutdown flag and the check if the entry is still running.
 
-		ZEN_CONSOLE("requesting shutdown of server on port {}", EntryPort);
+		ZEN_CONSOLE("requesting detached shutdown of server on port {}", EntryPort);
 		Entry->SignalShutdownRequest();
 
 		Stopwatch Timer;
-		while (Timer.GetElapsedTimeMs() < 5000)
+		while (Timer.GetElapsedTimeMs() < 10000)
 		{
 			Instance.Sweep();
 			Entry = Instance.Lookup(EntryPort);
@@ -261,6 +304,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			Sleep(100);
 		}
 	}
+
 	if (m_ForceTerminate)
 	{
 		// Try to find the running executable by path name
diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h
index 365f26002..94f896b4e 100644
--- a/src/zen/cmds/up_cmd.h
+++ b/src/zen/cmds/up_cmd.h
@@ -19,8 +19,9 @@ public:
 
 private:
 	cxxopts::Options	  m_Options{"up", "Bring up zen service"};
-	uint16_t			  m_Port	 = 0;
-	int					  m_OwnerPid = 0;
+	uint16_t			  m_Port		= 0;
+	bool				  m_ShowConsole = false;
+	bool				  m_ShowLog		= false;
 	std::filesystem::path m_ProgramBaseDir;
 };
 
diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h
index 48a2eb0d7..d1394cd9a 100644
--- a/src/zencore/include/zencore/process.h
+++ b/src/zencore/include/zencore/process.h
@@ -27,6 +27,7 @@ public:
 	ZENCORE_API [[nodiscard]] bool IsRunning() const;
 	ZENCORE_API [[nodiscard]] bool IsValid() const;
 	ZENCORE_API bool			   Wait(int TimeoutMs = -1);
+	ZENCORE_API bool			   Wait(int TimeoutMs, std::error_code& OutEc);
 	ZENCORE_API int				   WaitExitCode();
 	ZENCORE_API int				   GetExitCode();
 	ZENCORE_API bool			   Terminate(int ExitCode);
diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h
index 9362802a1..8fb781571 100644
--- a/src/zencore/include/zencore/thread.h
+++ b/src/zencore/include/zencore/thread.h
@@ -138,8 +138,8 @@ public:
 	ZENCORE_API explicit NamedEvent(std::string_view EventName);
 	ZENCORE_API ~NamedEvent();
 	ZENCORE_API void Close();
-	ZENCORE_API void Set();
-	ZENCORE_API bool Wait(int TimeoutMs = -1);
+	ZENCORE_API std::error_code Set();
+	ZENCORE_API bool			Wait(int TimeoutMs = -1);
 
 	NamedEvent(NamedEvent&& Rhs) noexcept : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; }
 
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index 88eb8af0d..f6b2cad57 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -56,7 +56,7 @@ const bool bNoZombieChildren = []() {
 }();
 
 static char
-GetPidStatus(int Pid)
+GetPidStatus(int Pid, std::error_code& OutEc)
 {
 	std::filesystem::path EntryPath = std::filesystem::path("/proc") / fmt::format("{}", Pid);
 	std::filesystem::path StatPath	= EntryPath / "stat";
@@ -80,11 +80,53 @@ GetPidStatus(int Pid)
 				}
 			}
 		}
+		else
+		{
+			OutEc = MakeErrorCodeFromLastError();
+		}
 	}
 	return 0;
 }
 
-#endif
+bool
+IsZombieProcess(int pid, std::error_code& OutEc)
+{
+	char Status = GetPidStatus(pid, OutEc);
+	if (OutEc)
+	{
+		return false;
+	}
+	if (Status == 'Z' || Status == 0)
+	{
+		return true;
+	}
+	return false;
+}
+
+#endif	// ZEN_PLATFORM_LINUX
+
+#if ZEN_PLATFORM_MAC
+bool
+IsZombieProcess(int pid, std::error_code& OutEc)
+{
+	struct kinfo_proc Info;
+	int				  Mib[4]   = {CTL_KERN, KERN_PROC, KERN_PROC_PID, pid};
+	size_t			  InfoSize = sizeof Info;
+
+	int Res = sysctl(Mib, 4, &Info, &InfoSize, NULL, 0);
+	if (Res != 0)
+	{
+		OutEc = MakeErrorCodeFromLastError();
+		return false;
+	}
+	if (Info.kp_proc.p_stat == SZOMB)
+	{
+		// Zombie process
+		return true;
+	}
+	return false;
+}
+#endif	// ZEN_PLATFORM_MAC
 
 ProcessHandle::ProcessHandle() = default;
 
@@ -220,7 +262,7 @@ ProcessHandle::Reset()
 }
 
 bool
-ProcessHandle::Wait(int TimeoutMs)
+ProcessHandle::Wait(int TimeoutMs, std::error_code& OutEc)
 {
 	using namespace std::literals;
 
@@ -237,20 +279,39 @@ ProcessHandle::Wait(int TimeoutMs)
 		case WAIT_TIMEOUT:
 			return false;
 
+		case WAIT_ABANDONED_0:
+			return true;
+
 		case WAIT_FAILED:
 			break;
 	}
-#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+
+	OutEc = MakeErrorCodeFromLastError();
+
+	return false;
+#endif	// ZEN_PLATFORM_WINDOWS
+
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 	const int SleepMs	= 20;
 	timespec  SleepTime = {0, SleepMs * 1000 * 1000};
 	for (int SleepedTimeMS = 0;; SleepedTimeMS += SleepMs)
 	{
 		int WaitState = 0;
-		waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED);
+		if (waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED) != -1)
+		{
+			if (WIFEXITED(WaitState))
+			{
+				m_ExitCode = WEXITSTATUS(WaitState);
+			}
+		}
 
-		if (WIFEXITED(WaitState))
+		if (!IsProcessRunning(m_Pid, OutEc))
+		{
+			return true;
+		}
+		else if (OutEc)
 		{
-			m_ExitCode = WEXITSTATUS(WaitState);
+			return false;
 		}
 
 		if (kill(m_Pid, 0) < 0)
@@ -260,7 +321,13 @@ ProcessHandle::Wait(int TimeoutMs)
 			{
 				return true;
 			}
-			ThrowSystemError(static_cast<uint32_t>(LastError), "Process::Wait kill failed"sv);
+			OutEc = MakeErrorCode(LastError);
+			return false;
+		}
+		else if (IsZombieProcess(m_Pid, OutEc))
+		{
+			ZEN_INFO("Found process {} in zombie state, treating as not running", m_Pid);
+			return true;
 		}
 
 		if (TimeoutMs >= 0 && SleepedTimeMS >= TimeoutMs)
@@ -270,10 +337,23 @@ ProcessHandle::Wait(int TimeoutMs)
 
 		nanosleep(&SleepTime, nullptr);
 	}
-#endif
+	return false;
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+}
 
-	// What might go wrong here, and what is meaningful to act on?
-	ThrowLastError("Process::Wait failed"sv);
+bool
+ProcessHandle::Wait(int TimeoutMs)
+{
+	std::error_code Ec;
+	if (Wait(TimeoutMs, Ec) && !Ec)
+	{
+		return true;
+	}
+	else if (Ec)
+	{
+		throw std::system_error(Ec, std::string("Process::Wait kill failed"));
+	}
+	return false;
 }
 
 int
@@ -740,34 +820,16 @@ IsProcessRunning(int pid, std::error_code& OutEc)
 	int Res		  = kill(pid_t(pid), 0);
 	if (Res == 0)
 	{
-#	if ZEN_PLATFORM_MAC
-		struct kinfo_proc Info;
-		int				  Mib[4]   = {CTL_KERN, KERN_PROC, KERN_PROC_PID, pid};
-		size_t			  InfoSize = sizeof Info;
-
-		int Res = sysctl(Mib, 4, &Info, &InfoSize, NULL, 0);
-		if (Res != 0)
-		{
-			int Error = errno;
-			OutEc	  = MakeErrorCode(Error);
-			return false;
-		}
-		ZEN_INFO("Found process {} with status {}", pid, (int)Info.kp_proc.p_stat);
-		if (Info.kp_proc.p_stat == SZOMB)
+		if (IsZombieProcess(pid, OutEc))
 		{
-			// Zombie process
+			ZEN_INFO("Found process {} in zombie state, treating as not running", pid);
 			return false;
 		}
-		return true;
-#	endif	// ZEN_PLATFORM_MAC
-#	if ZEN_PLATFORM_LINUX
-		char Status = GetPidStatus(pid);
-		if (Status == 'Z' || Status == 0)
+		if (OutEc)
 		{
 			return false;
 		}
 		return true;
-#	endif	// ZEN_PLATFORM_LINUX
 	}
 	int Error = errno;
 	if (Error == ESRCH)	 // No such process
@@ -969,11 +1031,14 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
 		{
 			if (EntryPath == ExecutableImage)
 			{
-				char Status = GetPidStatus(Pid);
-				if (Status && (Status != 'Z'))
+				char Status = GetPidStatus(Pid, Ec);
+				if (!Ec)
 				{
-					OutHandle.Initialize(Pid, Ec);
-					return Ec;
+					if (Status && (Status != 'Z'))
+					{
+						OutHandle.Initialize(Pid, Ec);
+						return Ec;
+					}
 				}
 			}
 		}
diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp
index 329e17eea..394197b8e 100644
--- a/src/zencore/thread.cpp
+++ b/src/zencore/thread.cpp
@@ -362,16 +362,23 @@ NamedEvent::Close()
 	m_EventHandle = nullptr;
 }
 
-void
+std::error_code
 NamedEvent::Set()
 {
 	ZEN_ASSERT(m_EventHandle != nullptr);
 #if ZEN_PLATFORM_WINDOWS
-	SetEvent(m_EventHandle);
+	if (!SetEvent(m_EventHandle))
+	{
+		return MakeErrorCodeFromLastError();
+	}
 #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 	int Sem = int(intptr_t(m_EventHandle) >> 32);
-	semctl(Sem, 0, SETVAL, 0);
+	if (semctl(Sem, 0, SETVAL, 0) == -1)
+	{
+		return MakeErrorCodeFromLastError();
+	}
 #endif
+	return {};
 }
 
 bool
diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h
index 8aa4a2773..6295ab5c5 100644
--- a/src/zenutil/include/zenutil/zenserverprocess.h
+++ b/src/zenutil/include/zenutil/zenserverprocess.h
@@ -67,9 +67,10 @@ struct ZenServerInstance
 	~ZenServerInstance();
 
 	int				   Shutdown();
-	bool			   SignalShutdown();
+	bool			   SignalShutdown(std::error_code& OutEc);
 	uint16_t		   WaitUntilReady();
 	[[nodiscard]] bool WaitUntilReady(int Timeout);
+	[[nodiscard]] bool WaitUntilExited(int Timeout, std::error_code& OutEc);
 	void			   EnableTermination() { m_Terminate = true; }
 	void			   DisableShutdownOnDestroy() { m_ShutdownOnDestroy = false; }
 	void			   Detach();
@@ -102,6 +103,8 @@ struct ZenServerInstance
 		SpawnServer(BasePort, AdditionalServerArgs, WaitTimeoutMs);
 	}
 
+	void SpawnServer(std::string_view ServerArgs, bool OpenConsole, int WaitTimeoutMs);
+
 	void		AttachToRunningServer(int BasePort = 0);
 	std::string GetBaseUri() const;
 	uint16_t	GetBasePort() const { return m_BasePort; }
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 29d590d00..7f053747e 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -574,12 +574,12 @@ ZenServerInstance::~ZenServerInstance()
 }
 
 bool
-ZenServerInstance::SignalShutdown()
+ZenServerInstance::SignalShutdown(std::error_code& OutEc)
 {
 	if (m_ShutdownEvent)
 	{
-		m_ShutdownEvent->Set();
-		return true;
+		OutEc = m_ShutdownEvent->Set();
+		return !OutEc;
 	}
 	else
 	{
@@ -604,26 +604,53 @@ ZenServerInstance::Shutdown()
 			}
 			else
 			{
-				if (m_Process.IsRunning())
+				if (!m_Process.IsRunning())
+				{
+					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
+					int ExitCode = m_Process.GetExitCode();
+					m_Process.Reset();
+					return ExitCode;
+				}
+
+				ZEN_DEBUG("Requesting zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
+				std::error_code Ec;
+				if (SignalShutdown(Ec))
 				{
-					ZEN_DEBUG("Requesting zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
-					if (!SignalShutdown())
-					{
-						ZEN_INFO("Terminating zenserver process as we did not wait for it to get ready {}", m_Name);
-						int ExitCode = 111;
-						m_Process.Terminate(ExitCode);
-						ZEN_DEBUG("zenserver process {} ({}) terminated", m_Name, m_Process.Pid());
-						return ExitCode;
-					}
 					ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
 					while (!m_Process.Wait(1000))
 					{
+						if (!m_Process.IsValid())
+						{
+							ZEN_WARN("Wait abandoned by invalid process");
+							break;
+						}
+						if (!m_Process.IsRunning())
+						{
+							ZEN_WARN("Wait abandoned by exited process");
+							return 0;
+						}
 						ZEN_WARN("Waiting for zenserver process {} ({}) timed out", m_Name, m_Process.Pid());
 					}
+					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
+					int ExitCode = m_Process.GetExitCode();
+					m_Process.Reset();
+					return ExitCode;
 				}
-				ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
-				int ExitCode = m_Process.GetExitCode();
-				m_Process.Reset();
+				else if (Ec)
+				{
+					ZEN_WARN("Terminating zenserver proces as we failed to signal zenserver process {} ({}) to shut down. Reason: '{}'",
+							 m_Name,
+							 m_Process.Pid(),
+							 Ec.message());
+				}
+				else
+				{
+					ZEN_INFO("Terminating zenserver process as we did not wait for it to get ready {}", m_Name);
+				}
+
+				int ExitCode = 111;
+				m_Process.Terminate(ExitCode);
+				ZEN_DEBUG("zenserver process {} ({}) terminated", m_Name, m_Process.Pid());
 				return ExitCode;
 			}
 		}
@@ -644,11 +671,10 @@ ZenServerInstance::Shutdown()
 }
 
 void
-ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs)
+ZenServerInstance::SpawnServer(std::string_view ServerArgs, bool OpenConsole, int WaitTimeoutMs)
 {
 	ZEN_ASSERT(!m_Process.IsValid());  // Only spawn once
 
-	const int MyPid	  = zen::GetCurrentProcessId();
 	const int ChildId = ++ChildIdCounter;
 
 	ExtendableStringBuilder<32> ChildEventName;
@@ -664,50 +690,11 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 
 	const bool IsTest = m_Env.IsTestEnvironment();
 
-	if (IsTest)
-	{
-		if (!m_OwnerPid.has_value())
-		{
-			m_OwnerPid = MyPid;
-		}
-
-		CommandLine << " --test --log-id " << m_Name;
-		CommandLine << " --no-sentry";
-
-		if (AdditionalServerArgs.find("--system-dir") == std::string_view::npos)
-		{
-			CommandLine << " --system-dir ";
-			PathToUtf8((m_Env.CreateNewTestDir() / "system-dir").c_str(), CommandLine);
-		}
-	}
-
-	if (m_OwnerPid.has_value())
-	{
-		CommandLine << " --owner-pid " << m_OwnerPid.value();
-	}
-
 	CommandLine << " --child-id " << ChildEventName;
 
-	if (std::string_view ServerClass = m_Env.GetServerClass(); ServerClass.empty() == false)
-	{
-		CommandLine << " --http " << ServerClass;
-	}
-
-	if (BasePort)
+	if (!ServerArgs.empty())
 	{
-		CommandLine << " --port " << BasePort;
-		m_BasePort = gsl::narrow_cast<uint16_t>(BasePort);
-	}
-
-	if (!m_TestDir.empty())
-	{
-		CommandLine << " --data-dir ";
-		PathToUtf8(m_TestDir.c_str(), CommandLine);
-	}
-
-	if (!AdditionalServerArgs.empty())
-	{
-		CommandLine << " " << AdditionalServerArgs;
+		CommandLine << " " << ServerArgs;
 	}
 
 	std::filesystem::path CurrentDirectory = std::filesystem::current_path();
@@ -715,22 +702,31 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	ZEN_DEBUG("Spawning server '{}'", LogId);
 
 	uint32_t CreationFlags = 0;
-	if (!IsTest)
+	if (OpenConsole)
 	{
 		CreationFlags |= CreateProcOptions::Flag_NewConsole;
 	}
 
-	const std::filesystem::path BaseDir		  = m_Env.ProgramBaseDir();
-	const std::filesystem::path Executable	  = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
-	const std::filesystem::path OutputPath	  = std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log");
-	CreateProcOptions			CreateOptions = {.WorkingDirectory = &CurrentDirectory, .Flags = CreationFlags, .StdoutFile = OutputPath};
-	CreateProcResult			ChildPid	  = CreateProc(Executable, CommandLine.ToView(), CreateOptions);
+	const std::filesystem::path BaseDir	   = m_Env.ProgramBaseDir();
+	const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
+	const std::filesystem::path OutputPath =
+		OpenConsole ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log");
+	CreateProcOptions CreateOptions = {.WorkingDirectory = &CurrentDirectory, .Flags = CreationFlags, .StdoutFile = OutputPath};
+	CreateProcResult  ChildPid		= CreateProc(Executable, CommandLine.ToView(), CreateOptions);
 #if ZEN_PLATFORM_WINDOWS
-	if (!ChildPid && ::GetLastError() == ERROR_ELEVATION_REQUIRED)
+	if (!ChildPid)
 	{
-		ZEN_DEBUG("Regular spawn failed - spawning elevated server");
-		CreateOptions.Flags |= CreateProcOptions::Flag_Elevated;
-		ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions);
+		DWORD Error = GetLastError();
+		if (Error == ERROR_ELEVATION_REQUIRED)
+		{
+			ZEN_DEBUG("Regular spawn failed - spawning elevated server");
+			CreateOptions.Flags |= CreateProcOptions::Flag_Elevated;
+			ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions);
+		}
+		else
+		{
+			ThrowSystemError(Error, "Server spawn failed");
+		}
 	}
 #endif
 
@@ -764,6 +760,70 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 }
 
 void
+ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs)
+{
+	ZEN_ASSERT(!m_Process.IsValid());  // Only spawn once
+
+	const int MyPid	  = zen::GetCurrentProcessId();
+	const int ChildId = ++ChildIdCounter;
+
+	ExtendableStringBuilder<32> LogId;
+	LogId << "Zen" << ChildId;
+	m_Name = LogId.ToString();
+
+	ExtendableStringBuilder<512> CommandLine;
+	CommandLine << "zenserver" ZEN_EXE_SUFFIX_LITERAL;	// see CreateProc() call for actual binary path
+
+	const bool IsTest = m_Env.IsTestEnvironment();
+
+	if (IsTest)
+	{
+		if (!m_OwnerPid.has_value())
+		{
+			m_OwnerPid = MyPid;
+		}
+
+		CommandLine << " --test --log-id " << m_Name;
+		CommandLine << " --no-sentry";
+
+		if (AdditionalServerArgs.find("--system-dir") == std::string_view::npos)
+		{
+			CommandLine << " --system-dir ";
+			PathToUtf8((m_Env.CreateNewTestDir() / "system-dir").c_str(), CommandLine);
+		}
+	}
+
+	if (m_OwnerPid.has_value())
+	{
+		CommandLine << " --owner-pid " << m_OwnerPid.value();
+	}
+
+	if (std::string_view ServerClass = m_Env.GetServerClass(); ServerClass.empty() == false)
+	{
+		CommandLine << " --http " << ServerClass;
+	}
+
+	if (BasePort)
+	{
+		CommandLine << " --port " << BasePort;
+		m_BasePort = gsl::narrow_cast<uint16_t>(BasePort);
+	}
+
+	if (!m_TestDir.empty())
+	{
+		CommandLine << " --data-dir ";
+		PathToUtf8(m_TestDir.c_str(), CommandLine);
+	}
+
+	if (!AdditionalServerArgs.empty())
+	{
+		CommandLine << " " << AdditionalServerArgs;
+	}
+
+	SpawnServer(CommandLine, !IsTest, WaitTimeoutMs);
+}
+
+void
 ZenServerInstance::CreateShutdownEvent(int BasePort)
 {
 	ExtendableStringBuilder<32> ChildShutdownEventName;
@@ -877,6 +937,16 @@ ZenServerInstance::WaitUntilReady(int Timeout)
 	return true;
 }
 
+bool
+ZenServerInstance::WaitUntilExited(int Timeout, std::error_code& OutEc)
+{
+	if (m_Process.IsRunning())
+	{
+		return m_Process.Wait(Timeout, OutEc);
+	}
+	return false;
+}
+
 void
 ZenServerInstance::OnServerReady()
 {