Changed how sponsor processes are managed

We can now monitor more than one process and if a new process is started on the same port we will hand over the owner pid to the process which is already executing before exiting. Note that this is only done if there is actually already an owner process in the instance list.

4 files changed, 194 insertions, 25 deletions

diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 8baf57d7e..973ef874a 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -1491,4 +1491,66 @@ TEST_CASE("http.package")
 	CHECK_EQ(ResponsePackage, TestPackage);
 }
 
+#	if 0
+TEST_CASE("lifetime.owner")
+{
+	// This test is designed to verify that the hand-over of sponsor processes is handled
+	// correctly for the case when a second or third process is launched on the same port
+	//
+	// Due to the nature of it, it cannot be
+
+	const uint16_t PortNumber = 23456;
+
+	ZenServerInstance	  Zen1(TestEnv);
+	std::filesystem::path TestDir1 = TestEnv.CreateNewTestDir();
+	Zen1.SetTestDir(TestDir1);
+	Zen1.SpawnServer(PortNumber);
+	Zen1.WaitUntilReady();
+	Zen1.Detach();
+
+	ZenServerInstance	  Zen2(TestEnv);
+	std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir();
+	Zen2.SetTestDir(TestDir2);
+	Zen2.SpawnServer(PortNumber);
+	Zen2.WaitUntilReady();
+	Zen2.Detach();
+}
+
+TEST_CASE("lifetime.owner.2")
+{
+	// This test is designed to verify that the hand-over of sponsor processes is handled
+	// correctly for the case when a second or third process is launched on the same port
+	//
+	// Due to the nature of it, it cannot be
+
+	const uint16_t PortNumber = 13456;
+
+	std::filesystem::path TestDir1 = TestEnv.CreateNewTestDir();
+	std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir();
+
+	ZenServerInstance Zen1(TestEnv);
+	Zen1.SetTestDir(TestDir1);
+	Zen1.SpawnServer(PortNumber);
+	Zen1.WaitUntilReady();
+
+	ZenServerInstance Zen2(TestEnv);
+	Zen2.SetTestDir(TestDir2);
+	Zen2.SetOwnerPid(Zen1.GetPid());
+	Zen2.SpawnServer(PortNumber + 1);
+	Zen2.Detach();
+
+	ZenServerInstance Zen3(TestEnv);
+	Zen3.SetTestDir(TestDir2);
+	Zen3.SetOwnerPid(Zen1.GetPid());
+	Zen3.SpawnServer(PortNumber + 1);
+	Zen3.Detach();
+
+	ZenServerInstance Zen4(TestEnv);
+	Zen4.SetTestDir(TestDir2);
+	Zen4.SetOwnerPid(Zen1.GetPid());
+	Zen4.SpawnServer(PortNumber + 1);
+	Zen4.Detach();
+}
+#	endif
+
 #endif
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 3b56d8683..aa4a42fd7 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -84,9 +84,12 @@
 
 class ZenServer
 {
+	ZenServerState::ZenServerEntry* m_ServerEntry = nullptr;
+
 public:
-	void Initialize(ZenServiceConfig& ServiceConfig, int BasePort, int ParentPid)
+	void Initialize(ZenServiceConfig& ServiceConfig, int BasePort, int ParentPid, ZenServerState::ZenServerEntry* ServerEntry)
 	{
+		m_ServerEntry = ServerEntry;
 		using namespace fmt::literals;
 		ZEN_INFO(ZEN_APP_NAME " initializing");
 
@@ -94,23 +97,29 @@ public:
 
 		if (ParentPid)
 		{
-			m_Process.Initialize(ParentPid);
+			zen::ProcessHandle OwnerProcess;
+			OwnerProcess.Initialize(ParentPid);
 
-			if (!m_Process.IsValid())
+			if (!OwnerProcess.IsValid())
 			{
 				ZEN_WARN("Unable to initialize process handle for specified parent pid #{}", ParentPid);
+
+				// If the pid is not reachable should we just shut down immediately? the intended owner process
+				// could have been killed or somehow crashed already
 			}
 			else
 			{
 				ZEN_INFO("Using parent pid #{} to control process lifetime", ParentPid);
 			}
+
+			m_ProcessMonitor.AddPid(ParentPid);
 		}
 
 		// Initialize/check mutex based on base port
 
 		std::string MutexName = "zen_{}"_format(BasePort);
 
-		if (zen::NamedMutex::Exists(MutexName) || (m_ServerMutex.Create(MutexName) == false))
+		if (zen::NamedMutex::Exists(MutexName) || ((m_ServerMutex.Create(MutexName) == false)))
 		{
 			throw std::runtime_error("Failed to create mutex '{}' - is another instance already running?"_format(MutexName).c_str());
 		}
@@ -267,7 +276,7 @@ public:
 
 	void Run()
 	{
-		if (m_Process.IsValid())
+		if (m_ProcessMonitor.IsActive())
 		{
 			EnqueueTimer();
 		}
@@ -282,7 +291,7 @@ public:
 			ZEN_INFO("        \\/   \\/     \\/          \\/                         \\/ ");
 		}
 
-		ZEN_INFO(ZEN_APP_NAME " now running");
+		ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", zen::GetCurrentProcessId());
 
 #if USE_SENTRY
 		sentry_clear_modulecache();
@@ -332,13 +341,33 @@ public:
 
 	void CheckOwnerPid()
 	{
-		if (m_Process.IsRunning())
+		// Pick up any new "owner" processes
+
+		std::set<uint32_t> AddedPids;
+
+		for (auto& PidEntry : m_ServerEntry->SponsorPids)
+		{
+			if (uint32_t ThisPid = PidEntry.load(std::memory_order::memory_order_relaxed))
+			{
+				if (PidEntry.compare_exchange_strong(ThisPid, 0))
+				{
+					if (AddedPids.insert(ThisPid).second)
+					{
+						m_ProcessMonitor.AddPid(ThisPid);
+
+						ZEN_INFO("added process with pid #{} as a sponsor process", ThisPid);
+					}
+				}
+			}
+		}
+
+		if (m_ProcessMonitor.IsRunning())
 		{
 			EnqueueTimer();
 		}
 		else
 		{
-			ZEN_INFO(ZEN_APP_NAME " exiting since parent process id {} is gone", m_Process.Pid());
+			ZEN_INFO(ZEN_APP_NAME " exiting since sponsor processes are all gone");
 
 			RequestExit(0);
 		}
@@ -366,7 +395,7 @@ private:
 	std::jthread		  m_IoRunner;
 	asio::io_context	  m_IoContext;
 	asio::steady_timer	  m_PidCheckTimer{m_IoContext};
-	zen::ProcessHandle	  m_Process;
+	zen::ProcessMonitor	  m_ProcessMonitor;
 	zen::NamedMutex		  m_ServerMutex;
 
 	zen::Ref<zen::HttpServer>						 m_Http;
@@ -411,27 +440,41 @@ main(int argc, char* argv[])
 	auto _ = zen::MakeGuard([&] { sentry_close(); });
 #endif
 
-	// Prototype config system, let's see how this pans out
+	try
+	{
+		// Prototype config system, we'll see how this pans out
+		//
+		// TODO: we need to report any parse errors here
 
-	ParseServiceConfig(GlobalOptions.DataDir, /* out */ ServiceConfig);
+		ParseServiceConfig(GlobalOptions.DataDir, /* out */ ServiceConfig);
 
-	ZEN_INFO("zen cache server starting on port {}", GlobalOptions.BasePort);
+		ZEN_INFO("zen cache server starting on port {}", GlobalOptions.BasePort);
 
-	try
-	{
 		ZenServerState ServerState;
 		ServerState.Initialize();
 		ServerState.Sweep();
 
-		if (ZenServerState::ZenServerEntry* Entry = ServerState.Lookup(GlobalOptions.BasePort))
+		ZenServerState::ZenServerEntry* Entry = ServerState.Lookup(GlobalOptions.BasePort);
+
+		if (Entry)
 		{
 			// Instance already running for this port? Should double check pid
 
 			ZEN_WARN("Looks like there is already a process listening to this port (pid: {})", Entry->Pid);
+
+			if (GlobalOptions.OwnerPid)
+			{
+				Entry->AddSponsorProcess(GlobalOptions.OwnerPid);
+
+				std::exit(0);
+			}
 		}
-		else
+
+		Entry = ServerState.Register(GlobalOptions.BasePort);
+
+		if (GlobalOptions.OwnerPid)
 		{
-			ServerState.Register(GlobalOptions.BasePort);
+			Entry->AddSponsorProcess(GlobalOptions.OwnerPid);
 		}
 
 		std::unique_ptr<std::thread>	 ShutdownThread;
@@ -445,15 +488,17 @@ main(int argc, char* argv[])
 		Server.SetDataRoot(GlobalOptions.DataDir);
 		Server.SetTestMode(GlobalOptions.IsTest);
 		Server.SetDedicatedMode(GlobalOptions.IsDedicated);
-		Server.Initialize(ServiceConfig, GlobalOptions.BasePort, GlobalOptions.OwnerPid);
+		Server.Initialize(ServiceConfig, GlobalOptions.BasePort, GlobalOptions.OwnerPid, Entry);
 
 		// Monitor shutdown signals
 
 		ShutdownThread.reset(new std::thread{[&] {
 			ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}'", ShutdownEventName);
-			ShutdownEvent->Wait();
-			ZEN_INFO("shutdown signal received");
-			Server.RequestExit(0);
+			if (ShutdownEvent->Wait())
+			{
+				ZEN_INFO("shutdown signal received");
+				Server.RequestExit(0);
+			}
 		}});
 
 		// If we have a parent process, establish the mechanisms we need
diff --git a/zenutil/include/zenserverprocess.h b/zenutil/include/zenserverprocess.h
index b81d61c25..7fcacf788 100644
--- a/zenutil/include/zenserverprocess.h
+++ b/zenutil/include/zenserverprocess.h
@@ -10,6 +10,7 @@
 
 #include <atomic>
 #include <filesystem>
+#include <optional>
 
 class ZenServerEnvironment
 {
@@ -44,6 +45,9 @@ struct ZenServerInstance
 	[[nodiscard]] bool WaitUntilReady(int Timeout);
 	void			   EnableTermination() { m_Terminate = true; }
 	void			   EnableMesh() { m_MeshEnabled = true; }
+	void			   Detach();
+	inline int		   GetPid() { return m_Process.Pid(); }
+	inline void		   SetOwnerPid(int Pid) { m_OwnerPid = Pid; }
 
 	void SetTestDir(std::filesystem::path TestDir)
 	{
@@ -66,6 +70,7 @@ private:
 	std::filesystem::path m_TestDir;
 	bool				  m_MeshEnabled = false;
 	int					  m_BasePort	= 0;
+	std::optional<int>	  m_OwnerPid;
 
 	void CreateShutdownEvent(int BasePort);
 };
@@ -90,7 +95,9 @@ public:
 		std::atomic<uint16_t> ListenPort;
 		std::atomic<uint16_t> Flags;
 		uint8_t				  SessionId[12];
+		std::atomic<uint32_t> SponsorPids[32];
 		uint8_t				  Padding[12];
+		uint8_t				  Padding2[96];
 
 		enum class FlagsEnum : uint16_t
 		{
@@ -101,9 +108,10 @@ public:
 
 		void Reset();
 		void SignalShutdownRequest();
+		bool AddSponsorProcess(uint32_t Pid);
 	};
 
-	static_assert(sizeof(ZenServerEntry) == 32);
+	static_assert(sizeof(ZenServerEntry) == 256);
 
 	void						  Initialize();
 	[[nodiscard]] bool			  InitializeReadOnly();
@@ -115,6 +123,6 @@ public:
 private:
 	void*			m_hMapFile = nullptr;
 	ZenServerEntry* m_Data;
-	int				m_MaxEntryCount = 4096 / sizeof(ZenServerEntry);
+	int				m_MaxEntryCount = 131072 / sizeof(ZenServerEntry);
 	ZenServerEntry* m_OurEntry		= nullptr;
 };
diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp
index 00f5d4c0d..5142c6a54 100644
--- a/zenutil/zenserverprocess.cpp
+++ b/zenutil/zenserverprocess.cpp
@@ -6,6 +6,7 @@
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
+#include <zencore/session.h>
 #include <zencore/string.h>
 
 #include <atlbase.h>
@@ -199,6 +200,9 @@ ZenServerState::Register(int ListenPort)
 				Entry.Pid	= Pid;
 				Entry.Flags = 0;
 
+				const zen::Oid SesId = zen::GetSessionId();
+				memcpy(Entry.SessionId, &SesId, sizeof SesId);
+
 				return &Entry;
 			}
 		}
@@ -264,6 +268,30 @@ ZenServerState::ZenServerEntry::SignalShutdownRequest()
 	Flags |= uint16_t(FlagsEnum::kShutdownPlease);
 }
 
+bool
+ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd)
+{
+	for (std::atomic<uint32_t>& PidEntry : SponsorPids)
+	{
+		if (PidEntry.load(std::memory_order::memory_order_relaxed) == 0)
+		{
+			uint32_t Expected = 0;
+			if (PidEntry.compare_exchange_strong(Expected, uint16_t(PidToAdd)))
+			{
+				// Success!
+				return true;
+			}
+		}
+		else if (PidEntry.load(std::memory_order::memory_order_relaxed) == PidToAdd)
+		{
+			// Success, the because pid is already in the list
+			return true;
+		}
+	}
+
+	return false;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 std::atomic<int> TestCounter{0};
@@ -395,7 +423,17 @@ ZenServerInstance::SpawnServer(int BasePort)
 
 	if (IsTest)
 	{
-		CommandLine << " --test --owner-pid " << MyPid << " --log-id " << LogId;
+		if (!m_OwnerPid.has_value())
+		{
+			m_OwnerPid = MyPid;
+		}
+
+		CommandLine << " --test --log-id " << LogId;
+	}
+
+	if (m_OwnerPid.has_value())
+	{
+		CommandLine << " --owner-pid " << m_OwnerPid.value();
 	}
 
 	CommandLine << " --child-id " << ChildEventName;
@@ -549,9 +587,25 @@ ZenServerInstance::AttachToRunningServer(int BasePort)
 }
 
 void
+ZenServerInstance::Detach()
+{
+	if (m_Process.IsValid())
+	{
+		m_Process.Reset();
+		m_ShutdownEvent.Close();
+	}
+}
+
+void
 ZenServerInstance::WaitUntilReady()
 {
-	m_ReadyEvent.Wait();
+	while (m_ReadyEvent.Wait(100) == false)
+	{
+		if (!m_Process.IsRunning() || !m_Process.IsValid())
+		{
+			return;
+		}
+	}
 }
 
 bool