zen startup hardening (#49)

- Feature: `zen up` command improvements
	- --`port` allows you to specify a base port when starting an instance
	- --`base-dir` allows you to specify a base directory for the zenserver executable if it is not located next to the zen.exe executable
- Feature: `zen down`
	- --`port` allows you to specify a base port when shutting down an instance
	- --`base-dir` allows you to specify a base directory for the zenserver executable if it is not located next to the zen.exe executable
	- --`force` if regular shutdown fails it tries to find a running zenserver.exe process and terminate it
	- If it fails to attach to the running server it now waits for it to exit when setting the RequestExit shared memory flag
- Improvement: zenserver now checks the RequestExit flag in the shared memory and exist gracefully if it is set
- Improvement: When adding a sponsor process to a running zenserver instance, we wait for it to be picked up from the shared memory section to determine success/fail

10 files changed, 494 insertions, 83 deletions

diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp
index 14f954064..61a901fd6 100644
--- a/src/zen/cmds/up_cmd.cpp
+++ b/src/zen/cmds/up_cmd.cpp
@@ -5,6 +5,7 @@
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
 #include <zencore/process.h>
+#include <zencore/timer.h>
 #include <zenutil/zenserverprocess.h>
 
 #include <memory>
@@ -13,13 +14,14 @@ namespace zen {
 
 UpCommand::UpCommand()
 {
+	m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("0"), "<hostport>");
 	m_Options.add_option("lifetime",
 						 "",
 						 "owner-pid",
 						 "Specify owning process id",
 						 cxxopts::value(m_OwnerPid)->default_value("0"),
 						 "<identifier>");
-	m_Options.add_options()("config", "Path to Lua config file", cxxopts::value(m_ConfigFile));
+	m_Options.add_option("", "b", "base-dir", "Parent folder of server executable", cxxopts::value(m_ProgramBaseDir), "<directory>");
 }
 
 UpCommand::~UpCommand() = default;
@@ -27,7 +29,7 @@ UpCommand::~UpCommand() = default;
 int
 UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 {
-	ZEN_UNUSED(GlobalOptions, argc, argv);
+	ZEN_UNUSED(GlobalOptions);
 
 	if (!ParseOptions(argc, argv))
 	{
@@ -45,14 +47,19 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				uint16_t EffectivePort = 0;
 			};
 			std::vector<EntryInfo> RunningEntries;
-			State.Snapshot([&RunningEntries](const zen::ZenServerState::ZenServerEntry& Entry) {
-				RunningEntries.push_back(EntryInfo{.Pid			  = Entry.Pid.load(),
-												   .DesiredPort	  = Entry.DesiredListenPort.load(),
-												   .EffectivePort = Entry.EffectiveListenPort.load()});
+			State.Snapshot([&RunningEntries, DesiredPort = this->m_Port](const zen::ZenServerState::ZenServerEntry& Entry) {
+				uint32_t Pid = Entry.Pid.load();
+				if (IsProcessRunning(Pid) && (DesiredPort == 0 || Entry.DesiredListenPort.load() == DesiredPort))
+				{
+					RunningEntries.push_back(EntryInfo{.Pid			  = Pid,
+													   .DesiredPort	  = Entry.DesiredListenPort.load(),
+													   .EffectivePort = Entry.EffectiveListenPort.load()});
+				}
 			});
 			if (RunningEntries.size() > 0)
 			{
-				ZEN_CONSOLE("Zen server already running. First instance at port {}, pid {}",
+				ZEN_CONSOLE("Zen server already running with base port {}. First instance at port {}, pid {}",
+							RunningEntries[0].DesiredPort,
 							RunningEntries[0].EffectivePort,
 							RunningEntries[0].Pid);
 				return 0;
@@ -60,20 +67,19 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		}
 	}
 
-	std::filesystem::path ExePath = zen::GetRunningExecutablePath();
-	ZenServerEnvironment  ServerEnvironment;
-	ServerEnvironment.Initialize(ExePath.parent_path());
+	if (m_ProgramBaseDir.empty())
+	{
+		std::filesystem::path ExePath = zen::GetRunningExecutablePath();
+		m_ProgramBaseDir			  = ExePath.parent_path();
+	}
+	ZenServerEnvironment ServerEnvironment;
+	ServerEnvironment.Initialize(m_ProgramBaseDir);
 	ZenServerInstance Server(ServerEnvironment);
 	if (m_OwnerPid != 0)
 	{
 		Server.SetOwnerPid(m_OwnerPid);
 	}
-	std::string AdditionalArguments;
-	if (!m_ConfigFile.empty())
-	{
-		AdditionalArguments = fmt::format("--config {}", m_ConfigFile);
-	}
-	Server.SpawnServer(0, AdditionalArguments);
+	Server.SpawnServer(m_Port, GlobalOptions.PassthroughCommandLine);
 
 	int Timeout = 10000;
 
@@ -102,7 +108,7 @@ AttachCommand::~AttachCommand() = default;
 int
 AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 {
-	ZEN_UNUSED(GlobalOptions, argc, argv);
+	ZEN_UNUSED(GlobalOptions);
 
 	if (!ParseOptions(argc, argv))
 	{
@@ -111,6 +117,7 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	ZenServerState Instance;
 	Instance.Initialize();
+	Instance.Sweep();
 	ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port);
 	if (!Entry)
 	{
@@ -132,7 +139,9 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 DownCommand::DownCommand()
 {
-	m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("8558"), "<hostport>");
+	m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("0"), "<hostport>");
+	m_Options.add_option("", "f", "force", "Force terminate if graceful shutdown fails", cxxopts::value(m_ForceTerminate), "<force>");
+	m_Options.add_option("", "b", "base-dir", "Parent folder of server executable", cxxopts::value(m_ProgramBaseDir), "<directory>");
 }
 
 DownCommand::~DownCommand() = default;
@@ -146,48 +155,103 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	{
 		return 0;
 	}
-	// Discover executing instances
 
+	// Discover executing instances
 	ZenServerState Instance;
 	Instance.Initialize();
 	ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port);
 
-	if (!Entry)
+	if (m_ProgramBaseDir.empty())
 	{
-		ZEN_WARN("no zen server to bring down");
-
-		return 0;
+		std::filesystem::path ExePath = zen::GetRunningExecutablePath();
+		m_ProgramBaseDir			  = ExePath.parent_path();
 	}
 
-	try
+	if (Entry)
 	{
-		std::filesystem::path ExePath = zen::GetRunningExecutablePath();
+		int			   EntryPort		= (int)Entry->DesiredListenPort.load();
+		const uint32_t ServerProcessPid = Entry->Pid.load();
+		try
+		{
+			ZenServerEnvironment ServerEnvironment;
+			ServerEnvironment.Initialize(m_ProgramBaseDir);
+			ZenServerInstance Server(ServerEnvironment);
+			Server.AttachToRunningServer(EntryPort);
 
-		ZenServerEnvironment ServerEnvironment;
-		ServerEnvironment.Initialize(ExePath.parent_path());
-		ZenServerInstance Server(ServerEnvironment);
-		Server.AttachToRunningServer(m_Port);
+			ZEN_CONSOLE("attached to server on port {} (pid {}), requesting shutdown", EntryPort, ServerProcessPid);
 
-		ZEN_CONSOLE("attached to server on port {}, requesting shutdown", m_Port);
+			Server.Shutdown();
 
-		Server.Shutdown();
+			ZEN_CONSOLE("shutdown complete");
 
-		ZEN_CONSOLE("shutdown complete");
+			return 0;
+		}
+		catch (const std::exception& Ex)
+		{
+			ZEN_DEBUG("Exception caught when requesting shutdown: {}", Ex.what());
+		}
 
-		return 0;
+		// Since we cannot obtain a handle to the process we are unable to block on the process
+		// handle to determine when the server has shut down. Thus we signal that we would like
+		// a shutdown via the shutdown flag and the check if the entry is still running.
+
+		ZEN_CONSOLE("requesting shutdown of server on port {}", EntryPort);
+		Entry->SignalShutdownRequest();
+
+		Stopwatch Timer;
+		while (Timer.GetElapsedTimeMs() < 5000)
+		{
+			Instance.Sweep();
+			Entry = Instance.Lookup(EntryPort);
+			if (Entry == nullptr)
+			{
+				ZEN_CONSOLE("shutdown complete");
+				return 0;
+			}
+			if (Entry->Pid.load() != ServerProcessPid)
+			{
+				ZEN_CONSOLE("shutdown complete");
+				return 0;
+			}
+			Sleep(100);
+		}
 	}
-	catch (const std::exception& Ex)
+	if (m_ForceTerminate)
 	{
-		ZEN_DEBUG("Exception caught when requesting shutdown: {}", Ex.what());
+		// Try to find the running executable by path name
+		std::filesystem::path ServerExePath = m_ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
+		ProcessHandle		  RunningProcess;
+		if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec)
+		{
+			ZEN_WARN("attempting hard terminate of zen process with pid ({})", RunningProcess.Pid());
+			try
+			{
+				if (RunningProcess.Terminate(0))
+				{
+					ZEN_CONSOLE("terminate complete");
+					return 0;
+				}
+				ZEN_CONSOLE("failed to terminate server, still running");
+				return 1;
+			}
+			catch (const std::exception& Ex)
+			{
+				ZEN_CONSOLE("failed to terminate server: '{}'", Ex.what());
+				return 1;
+			}
+		}
+		else
+		{
+			ZEN_CONSOLE("Failed to find process '{}', reason: {}", ServerExePath.string(), Ec.message());
+		}
+	}
+	else if (Entry)
+	{
+		ZEN_CONSOLE("failed to shutdown of server on port {}, use --force to hard terminate process", Entry->DesiredListenPort.load());
+		return 1;
 	}
 
-	// Since we cannot obtain a handle to the process we are unable to block on the process
-	// handle to determine when the server has shut down. Thus we signal that we would like
-	// a shutdown via the shutdown flag and then exit.
-
-	ZEN_CONSOLE("requesting shutdown of server on port {}", m_Port);
-	Entry->SignalShutdownRequest();
-
+	ZEN_CONSOLE("no zen server to bring down");
 	return 0;
 }
 
diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h
index 510cc865e..5e6af9d08 100644
--- a/src/zen/cmds/up_cmd.h
+++ b/src/zen/cmds/up_cmd.h
@@ -4,6 +4,8 @@
 
 #include "../zen.h"
 
+#include <filesystem>
+
 namespace zen {
 
 class UpCommand : public ZenCmdBase
@@ -16,10 +18,10 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{"up", "Bring up zen service"};
-	uint16_t		 m_Port		= 0;
-	int				 m_OwnerPid = 0;
-	std::string		 m_ConfigFile;
+	cxxopts::Options	  m_Options{"up", "Bring up zen service"};
+	uint16_t			  m_Port	 = 0;
+	int					  m_OwnerPid = 0;
+	std::filesystem::path m_ProgramBaseDir;
 };
 
 class AttachCommand : public ZenCmdBase
@@ -47,8 +49,10 @@ public:
 	virtual cxxopts::Options& Options() override { return m_Options; }
 
 private:
-	cxxopts::Options m_Options{"down", "Bring down zen service"};
-	uint16_t		 m_Port;
+	cxxopts::Options	  m_Options{"down", "Bring down zen service"};
+	uint16_t			  m_Port;
+	bool				  m_ForceTerminate = false;
+	std::filesystem::path m_ProgramBaseDir;
 };
 
 }  // namespace zen
diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h
index a14aa5a71..6dce2d264 100644
--- a/src/zencore/include/zencore/process.h
+++ b/src/zencore/include/zencore/process.h
@@ -28,7 +28,7 @@ public:
 	ZENCORE_API [[nodiscard]] bool IsValid() const;
 	ZENCORE_API bool			   Wait(int TimeoutMs = -1);
 	ZENCORE_API int				   WaitExitCode();
-	ZENCORE_API void			   Terminate(int ExitCode);
+	ZENCORE_API bool			   Terminate(int ExitCode);
 	ZENCORE_API void			   Reset();
 	[[nodiscard]] inline int	   Pid() const { return m_Pid; }
 
@@ -91,9 +91,12 @@ private:
 };
 
 ZENCORE_API bool IsProcessRunning(int pid);
+ZENCORE_API bool IsProcessRunning(int pid, std::error_code& OutEc);
 ZENCORE_API int	 GetCurrentProcessId();
 int				 GetProcessId(CreateProcResult ProcId);
 
+std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle);
+
 void process_forcelink();  // internal
 
 }  // namespace zen
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index d8be0d343..df2a87352 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -14,6 +14,7 @@
 #if ZEN_PLATFORM_WINDOWS
 #	include <shellapi.h>
 #	include <Shlobj.h>
+#	include <TlHelp32.h>
 #	include <zencore/windows.h>
 #else
 #	include <fcntl.h>
@@ -23,11 +24,16 @@
 #	include <sys/sem.h>
 #	include <sys/stat.h>
 #	include <sys/syscall.h>
+#	include <sys/sysctl.h>
 #	include <sys/wait.h>
 #	include <time.h>
 #	include <unistd.h>
 #endif
 
+#if ZEN_PLATFORM_MAC
+#	include <libproc.h>
+#endif
+
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/format.h>
 ZEN_THIRD_PARTY_INCLUDES_END
@@ -48,6 +54,36 @@ const bool bNoZombieChildren = []() {
 	sigaction(SIGCHLD, &Action, nullptr);
 	return true;
 }();
+
+static char
+GetPidStatus(int Pid)
+{
+	std::filesystem::path EntryPath = std::filesystem::path("/proc") / fmt::format("{}", Pid);
+	std::filesystem::path StatPath	= EntryPath / "stat";
+	if (std::filesystem::is_regular_file(StatPath))
+	{
+		FILE* StatFile = fopen(StatPath.c_str(), "r");
+		if (StatFile)
+		{
+			char Buffer[5120];
+			int	 Size = fread(Buffer, 1, 5120 - 1, StatFile);
+			fclose(StatFile);
+			if (Size > 0)
+			{
+				Buffer[Size + 1] = 0;
+				char* ScanPtr	 = strrchr(Buffer, ')');
+				if (ScanPtr && ScanPtr[1] != '\0')
+				{
+					ScanPtr += 2;
+					char State = *ScanPtr;
+					return State;
+				}
+			}
+		}
+	}
+	return 0;
+}
+
 #endif
 
 ProcessHandle::ProcessHandle() = default;
@@ -121,7 +157,8 @@ ProcessHandle::IsRunning() const
 	GetExitCodeProcess(m_ProcessHandle, &ExitCode);
 	bActive = (ExitCode == STILL_ACTIVE);
 #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	bActive = (kill(pid_t(m_Pid), 0) == 0);
+	std::error_code _;
+	bActive = IsProcessRunning(m_Pid, _);
 #endif
 
 	return bActive;
@@ -133,29 +170,40 @@ ProcessHandle::IsValid() const
 	return (m_ProcessHandle != nullptr);
 }
 
-void
+bool
 ProcessHandle::Terminate(int ExitCode)
 {
 	if (!IsRunning())
 	{
-		return;
+		return true;
 	}
 
-	bool bSuccess = false;
-
 #if ZEN_PLATFORM_WINDOWS
-	TerminateProcess(m_ProcessHandle, ExitCode);
+	BOOL bTerminated = TerminateProcess(m_ProcessHandle, ExitCode);
+	if (!bTerminated)
+	{
+		return false;
+	}
 	DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, INFINITE);
-	bSuccess		 = (WaitResult != WAIT_OBJECT_0);
+	bool  bSuccess	 = (WaitResult == WAIT_OBJECT_0) || (WaitResult == WAIT_ABANDONED_0);
+	if (!bSuccess)
+	{
+		return false;
+	}
 #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 	ZEN_UNUSED(ExitCode);
-	bSuccess			= (kill(m_Pid, SIGKILL) == 0);
-#endif
-
-	if (!bSuccess)
+	int Res = kill(pid_t(m_Pid), SIGKILL);
+	if (Res != 0)
 	{
-		// What might go wrong here, and what is meaningful to act on?
+		int err = errno;
+		if (err != ESRCH)
+		{
+			return false;
+		}
 	}
+#endif
+	Reset();
+	return true;
 }
 
 void
@@ -649,7 +697,7 @@ ProcessMonitor::IsActive() const
 //////////////////////////////////////////////////////////////////////////
 
 bool
-IsProcessRunning(int pid)
+IsProcessRunning(int pid, std::error_code& OutEc)
 {
 	// This function is arguably not super useful, a pid can be re-used
 	// by the OS so holding on to a pid and polling it over some time
@@ -665,7 +713,8 @@ IsProcessRunning(int pid)
 		{
 			return false;
 		}
-		ThrowSystemError(Error, fmt::format("failed to open process with pid {}", pid));
+		OutEc = MakeErrorCode(Error);
+		return false;
 	}
 	auto _ = MakeGuard([hProc]() { CloseHandle(hProc); });
 
@@ -678,27 +727,72 @@ IsProcessRunning(int pid)
 	else
 	{
 		DWORD Error = GetLastError();
-		ThrowSystemError(Error, fmt::format("failed to get process exit code for pid {}", pid));
+		OutEc		= MakeErrorCode(Error);
+		return false;
 	}
 	return bStillActive;
 #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 	int Res		  = kill(pid_t(pid), 0);
 	if (Res == 0)
 	{
+#	if ZEN_PLATFORM_MAC
+		struct kinfo_proc Info;
+		int				  Mib[4]   = {CTL_KERN, KERN_PROC, KERN_PROC_PID, pid};
+		size_t			  InfoSize = sizeof Info;
+
+		int Res = sysctl(Mib, 4, &Info, &InfoSize, NULL, 0);
+		if (Res != 0)
+		{
+			int Error = errno;
+			OutEc	  = MakeErrorCode(Error);
+			return false;
+		}
+		ZEN_INFO("Found process {} with status {}", pid, (int)Info.kp_proc.p_stat);
+		if (Info.kp_proc.p_stat == SZOMB)
+		{
+			// Zombie process
+			return false;
+		}
+		return true;
+#	endif	// ZEN_PLATFORM_MAC
+#	if ZEN_PLATFORM_LINUX
+		char Status = GetPidStatus(pid);
+		if (Status == 'Z' || Status == 0)
+		{
+			return false;
+		}
 		return true;
+#	endif	// ZEN_PLATFORM_LINUX
 	}
 	int Error = errno;
 	if (Error == ESRCH)	 // No such process
 	{
 		return false;
 	}
+	else if (Error == ENOENT)
+	{
+		return false;
+	}
 	else
 	{
-		ThrowSystemError(Error, fmt::format("Failed to signal running process %d: %d", pid, Error));
+		OutEc = MakeErrorCode(Error);
+		return false;
 	}
 #endif
 }
 
+bool
+IsProcessRunning(int pid)
+{
+	std::error_code Ec;
+	bool			IsRunning = IsProcessRunning(pid, Ec);
+	if (Ec)
+	{
+		ThrowSystemError(Ec.value(), fmt::format("Failed determining if process with pid {} is running", pid));
+	}
+	return IsRunning;
+}
+
 int
 GetCurrentProcessId()
 {
@@ -719,6 +813,129 @@ GetProcessId(CreateProcResult ProcId)
 #endif
 }
 
+std::error_code
+FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle)
+{
+#if ZEN_PLATFORM_WINDOWS
+	HANDLE ProcessSnapshotHandle = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
+	if (ProcessSnapshotHandle == INVALID_HANDLE_VALUE)
+	{
+		return MakeErrorCodeFromLastError();
+	}
+	auto _ = MakeGuard([&]() { CloseHandle(ProcessSnapshotHandle); });
+
+	PROCESSENTRY32 Entry;
+	Entry.dwSize = sizeof(PROCESSENTRY32);
+	if (Process32First(ProcessSnapshotHandle, (LPPROCESSENTRY32)&Entry))
+	{
+		do
+		{
+			if (ExecutableImage.filename() == Entry.szExeFile)
+			{
+				HANDLE ModuleSnapshotHandle = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, Entry.th32ProcessID);
+				if (ModuleSnapshotHandle != INVALID_HANDLE_VALUE)
+				{
+					auto		  __ = MakeGuard([&]() { CloseHandle(ModuleSnapshotHandle); });
+					MODULEENTRY32 ModuleEntry;
+					ModuleEntry.dwSize = sizeof(MODULEENTRY32);
+					if (Module32First(ModuleSnapshotHandle, (LPMODULEENTRY32)&ModuleEntry))
+					{
+						std::filesystem::path EntryPath(ModuleEntry.szExePath);
+						if (EntryPath == ExecutableImage)
+						{
+							HANDLE Handle =
+								OpenProcess(PROCESS_TERMINATE | SYNCHRONIZE | PROCESS_QUERY_INFORMATION, FALSE, Entry.th32ProcessID);
+							if (Handle == NULL)
+							{
+								return MakeErrorCodeFromLastError();
+							}
+							OutHandle.Initialize((void*)Handle);
+							return {};
+						}
+					}
+				}
+			}
+		} while (::Process32Next(ProcessSnapshotHandle, (LPPROCESSENTRY32)&Entry));
+	}
+	return MakeErrorCodeFromLastError();
+#endif	// ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_MAC
+	int	   Mib[4]	  = {CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0};
+	size_t BufferSize = 0;
+
+	struct kinfo_proc* Processes = nullptr;
+	uint32_t		   ProcCount = 0;
+
+	if (sysctl(Mib, 4, NULL, &BufferSize, NULL, 0) != -1 && BufferSize > 0)
+	{
+		struct kinfo_proc* Processes = (struct kinfo_proc*)malloc(BufferSize);
+		auto			   _		 = MakeGuard([&]() { free(Processes); });
+		if (sysctl(Mib, 4, Processes, &BufferSize, NULL, 0) != -1)
+		{
+			ProcCount = (uint32_t)(BufferSize / sizeof(struct kinfo_proc));
+			char Buffer[PROC_PIDPATHINFO_MAXSIZE];
+			for (uint32_t ProcIndex = 0; ProcIndex < ProcCount; ProcIndex++)
+			{
+				if (Processes[ProcIndex].kp_proc.p_stat != SZOMB)
+				{
+					pid_t Pid = Processes[ProcIndex].kp_proc.p_pid;
+					int	  Res = proc_pidpath(Pid, Buffer, sizeof(Buffer));
+					if (Res > 0)
+					{
+						std::filesystem::path EntryPath(Buffer);
+						if (EntryPath == ExecutableImage)
+						{
+							std::error_code Ec;
+							OutHandle.Initialize(Pid, Ec);
+							return Ec;
+						}
+					}
+				}
+			}
+		}
+	}
+	return MakeErrorCodeFromLastError();
+#endif	// ZEN_PLATFORM_MAC
+#if ZEN_PLATFORM_LINUX
+	std::vector<uint32_t> RunningPids;
+	DirectoryContent	  ProcList;
+	GetDirectoryContent("/proc", DirectoryContent::IncludeDirsFlag, ProcList);
+	for (const std::filesystem::path& EntryPath : ProcList.Directories)
+	{
+		std::string				EntryName = EntryPath.stem();
+		std::optional<uint32_t> Pid		  = ParseInt<uint32_t>(EntryName);
+		if (Pid.has_value())
+		{
+			RunningPids.push_back(Pid.value());
+		}
+	}
+
+	for (uint32_t Pid : RunningPids)
+	{
+		char Status = GetPidStatus(Pid);
+		if (Status && (Status != 'Z'))
+		{
+			std::filesystem::path EntryPath	  = std::filesystem::path("/proc") / fmt::format("{}", Pid);
+			std::filesystem::path ExeLinkPath = EntryPath / "exe";
+			char				  Link[4096];
+			ssize_t				  BytesRead = readlink(ExeLinkPath.c_str(), Link, sizeof(Link) - 1);
+			if (BytesRead > 0)
+			{
+				Link[BytesRead] = '\0';
+				std::filesystem::path ExePath(Link);
+				if (ExePath == ExecutableImage)
+				{
+					std::error_code Ec;
+					OutHandle.Initialize(Pid, Ec);
+					return Ec;
+				}
+			}
+		}
+	}
+	return {};
+#endif	// ZEN_PLATFORM_LINUX
+}
+
 #if ZEN_WITH_TESTS
 
 void
@@ -735,6 +952,14 @@ TEST_CASE("Process")
 	CHECK(IsProcessRunning(Pid));
 }
 
+TEST_CASE("FindProcess")
+{
+	ProcessHandle	Process;
+	std::error_code Ec = FindProcess(GetRunningExecutablePath(), Process);
+	CHECK(!Ec);
+	CHECK(Process.IsValid());
+}
+
 TEST_CASE("BuildArgV")
 {
 	const char* Words[] = {"one", "two", "three", "four", "five"};
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 3cf96f1cc..aa711ab7c 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -146,8 +146,7 @@ namespace zen::tests {
 TEST_CASE("default.single")
 {
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
-
-	ZenServerInstance Instance(TestEnv);
+	ZenServerInstance	  Instance(TestEnv);
 	Instance.SetTestDir(TestDir);
 	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady();
 
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 6b31dc82e..2d2b24bbb 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -103,21 +103,42 @@ ZenEntryPoint::Run()
 		ServerState.Initialize();
 		ServerState.Sweep();
 
-		ZenServerState::ZenServerEntry* Entry = ServerState.Lookup(m_ServerOptions.BasePort);
-
-		if (Entry)
+		uint32_t						AttachSponsorProcessRetriesLeft = 3;
+		ZenServerState::ZenServerEntry* Entry							= ServerState.Lookup(m_ServerOptions.BasePort);
+		while (Entry)
 		{
 			if (m_ServerOptions.OwnerPid)
 			{
+				if (!IsProcessRunning(m_ServerOptions.OwnerPid))
+				{
+					ZEN_WARN("Sponsor owner pid {} is no longer running, will not add sponsor to process listening to port {} (pid: {})",
+							 m_ServerOptions.OwnerPid,
+							 m_ServerOptions.BasePort,
+							 Entry->Pid.load());
+					std::exit(1);
+				}
 				ZEN_INFO(
 					"Looks like there is already a process listening to this port {} (pid: {}), attaching owner pid {} to running instance",
 					m_ServerOptions.BasePort,
 					Entry->Pid.load(),
 					m_ServerOptions.OwnerPid);
 
-				Entry->AddSponsorProcess(m_ServerOptions.OwnerPid);
-
-				std::exit(0);
+				if (Entry->AddSponsorProcess(m_ServerOptions.OwnerPid))
+				{
+					std::exit(0);
+				}
+				if (AttachSponsorProcessRetriesLeft-- > 0)
+				{
+					Entry = ServerState.Lookup(m_ServerOptions.BasePort);
+				}
+				else
+				{
+					ZEN_WARN("Failed to add sponsor owner pid {} to process listening to port {} (pid: {})",
+							 m_ServerOptions.OwnerPid,
+							 m_ServerOptions.BasePort,
+							 Entry->Pid.load());
+					std::exit(1);
+				}
 			}
 			else
 			{
@@ -329,6 +350,11 @@ main(int argc, char* argv[])
 		}
 	}
 
+#if ZEN_PLATFORM_LINUX | ZEN_PLATFORM_MAC
+	// Detach ourselves from any parent process
+	setsid();
+#endif
+
 	signal(SIGINT, utils::SignalCallbackHandler);
 	signal(SIGTERM, utils::SignalCallbackHandler);
 
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index e6e451952..86dbc9617 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -475,6 +475,8 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 
 		EnqueueStateMarkerTimer();
 	}
+
+	EnqueueStateExitFlagTimer();
 }
 
 void
@@ -805,6 +807,14 @@ ZenServer::EnqueueSigIntTimer()
 }
 
 void
+ZenServer::EnqueueStateExitFlagTimer()
+{
+	m_SigIntTimer.expires_after(std::chrono::milliseconds(500));
+	m_SigIntTimer.async_wait([this](const asio::error_code&) { CheckStateExitFlag(); });
+	EnsureIoRunner();
+}
+
+void
 ZenServer::EnqueueStatsReportingTimer()
 {
 	m_StatsReportingTimer.expires_after(std::chrono::milliseconds(500));
@@ -858,6 +868,17 @@ ZenServer::CheckSigInt()
 	EnqueueSigIntTimer();
 }
 
+void
+ZenServer::CheckStateExitFlag()
+{
+	if (m_ServerEntry && m_ServerEntry->IsShutdownRequested())
+	{
+		RequestExit(0);
+		return;
+	}
+	EnqueueStateExitFlagTimer();
+}
+
 bool
 ZenServer::UpdateProcessMonitor()
 {
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index dd259f855..550047a5d 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -74,9 +74,11 @@ public:
 	void EnqueueProcessMonitorTimer();
 	void EnqueueStateMarkerTimer();
 	void EnqueueSigIntTimer();
+	void EnqueueStateExitFlagTimer();
 	void EnqueueStatsReportingTimer();
 	void CheckStateMarker();
 	void CheckSigInt();
+	void CheckStateExitFlag();
 	void CheckOwnerPid();
 	bool UpdateProcessMonitor();
 	void ScrubStorage();
diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h
index 15138341c..720b30766 100644
--- a/src/zenutil/include/zenutil/zenserverprocess.h
+++ b/src/zenutil/include/zenutil/zenserverprocess.h
@@ -74,6 +74,7 @@ struct ZenServerInstance
 	inline int		   GetPid() { return m_Process.Pid(); }
 	inline void		   SetOwnerPid(int Pid) { m_OwnerPid = Pid; }
 	bool			   IsRunning();
+	bool			   Terminate();
 
 	void SetTestDir(std::filesystem::path TestDir);
 
@@ -160,7 +161,9 @@ public:
 		Oid	 GetSessionId() const { return Oid::FromMemory(SessionId); }
 		void Reset();
 		void SignalShutdownRequest();
+		bool IsShutdownRequested() const;
 		void SignalReady();
+		bool IsReady() const;
 		bool AddSponsorProcess(uint32_t Pid);
 	};
 
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 7725d0af6..384371df3 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -9,6 +9,7 @@
 #include <zencore/session.h>
 #include <zencore/string.h>
 #include <zencore/thread.h>
+#include <zencore/timer.h>
 
 #include <atomic>
 
@@ -240,11 +241,15 @@ ZenServerState::Lookup(int DesiredListenPort)
 {
 	for (int i = 0; i < m_MaxEntryCount; ++i)
 	{
-		if (m_Data[i].DesiredListenPort == DesiredListenPort)
+		uint16_t EntryPort = m_Data[i].DesiredListenPort;
+		if (EntryPort != 0)
 		{
-			if (IsProcessRunning(m_Data[i].Pid))
+			if (DesiredListenPort == 0 || (EntryPort == DesiredListenPort))
 			{
-				return &m_Data[i];
+				if (IsProcessRunning(m_Data[i].Pid))
+				{
+					return &m_Data[i];
+				}
 			}
 		}
 	}
@@ -352,6 +357,12 @@ ZenServerState::ZenServerEntry::SignalShutdownRequest()
 	Flags |= uint16_t(FlagsEnum::kShutdownPlease);
 }
 
+bool
+ZenServerState::ZenServerEntry::IsShutdownRequested() const
+{
+	return (Flags.load() & static_cast<uint16_t>(FlagsEnum::kShutdownPlease)) != 0;
+}
+
 void
 ZenServerState::ZenServerEntry::SignalReady()
 {
@@ -359,20 +370,43 @@ ZenServerState::ZenServerEntry::SignalReady()
 }
 
 bool
+ZenServerState::ZenServerEntry::IsReady() const
+{
+	return (Flags.load() & static_cast<uint16_t>(FlagsEnum::kIsReady)) != 0;
+}
+
+bool
 ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd)
 {
-	for (std::atomic<uint32_t>& PidEntry : SponsorPids)
+	uint32_t ServerPid	   = Pid.load();
+	auto	 WaitForPickup = [&](uint32_t AddedSlotIndex) {
+		Stopwatch Timer;
+		while (SponsorPids[AddedSlotIndex] == PidToAdd)
+		{
+			// Sponsor processes are checked every second, so 2 second wait time should be enough
+			if (Timer.GetElapsedTimeMs() > 2000)
+			{
+				return false;
+			}
+			if (!IsProcessRunning(ServerPid))
+			{
+				return false;
+			}
+			Sleep(100);
+		}
+		return true;
+	};
+	for (uint32_t SponsorIndex = 0; SponsorIndex < 8; SponsorIndex++)
 	{
-		if (PidEntry.load(std::memory_order_relaxed) == PidToAdd)
+		if (SponsorPids[SponsorIndex].load(std::memory_order_relaxed) == PidToAdd)
 		{
-			// Success, the because pid is already in the list
-			return true;
+			return WaitForPickup(SponsorIndex);
 		}
 		uint32_t Expected = 0;
-		if (PidEntry.compare_exchange_strong(Expected, PidToAdd))
+		if (SponsorPids[SponsorIndex].compare_exchange_strong(Expected, PidToAdd))
 		{
 			// Success!
-			return true;
+			return WaitForPickup(SponsorIndex);
 		}
 	}
 
@@ -661,7 +695,13 @@ ZenServerInstance::AttachToRunningServer(int BasePort)
 	}
 	else
 	{
-		State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) { Entry = &InEntry; });
+		State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) {
+			if (IsProcessRunning(InEntry.Pid.load()))
+			{
+				ZEN_INFO("Found entry pid {}, baseport {}", InEntry.Pid.load(), InEntry.DesiredListenPort.load());
+				Entry = &InEntry;
+			}
+		});
 	}
 
 	if (!Entry)
@@ -670,6 +710,8 @@ ZenServerInstance::AttachToRunningServer(int BasePort)
 		throw std::runtime_error("No server found");
 	}
 
+	ZEN_INFO("Found entry pid {}, baseport {}", Entry->Pid.load(), Entry->DesiredListenPort.load());
+
 	std::error_code Ec;
 	m_Process.Initialize(Entry->Pid, Ec);
 
@@ -786,4 +828,26 @@ ZenServerInstance::IsRunning()
 	return m_Process.IsRunning();
 }
 
+bool
+ZenServerInstance::Terminate()
+{
+	const std::filesystem::path BaseDir	   = m_Env.ProgramBaseDir();
+	const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
+	ProcessHandle				RunningProcess;
+	std::error_code				Ec = FindProcess(Executable, RunningProcess);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("failed to look up running server executable '{}'", Executable));
+	}
+	if (RunningProcess.IsValid())
+	{
+		if (RunningProcess.Terminate(0))
+		{
+			return true;
+		}
+		return false;
+	}
+	return true;
+}
+
 }  // namespace zen