harden attach sponsor process (#14)

- Improvement: Delay exiting due to no sponsor processes by one second to handle race conditions
- Improvement: Safer IsProcessRunning check
- Improvement: make sure we can RequestApplicationExit safely from any thread

3 files changed, 37 insertions, 25 deletions

diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index b4cb2464b..a9246ed8f 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -210,11 +210,8 @@ ZenEntryPoint::Run()
 
 			if (ShutdownEvent->Wait())
 			{
-				if (!IsApplicationExitRequested())
-				{
-					ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId());
-					Server.RequestExit(0);
-				}
+				ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId());
+				Server.RequestExit(0);
 			}
 			else
 			{
@@ -253,10 +250,7 @@ ZenEntryPoint::Run()
 	catch (std::exception& e)
 	{
 		ZEN_CRITICAL("Caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what());
-		if (!IsApplicationExitRequested())
-		{
-			RequestApplicationExit(1);
-		}
+		RequestApplicationExit(1);
 	}
 
 	ShutdownServerLogging();
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index daec743a0..d1faeb8b6 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -693,7 +693,6 @@ ZenServer::Run()
 	if (m_IsPowerCycle)
 	{
 		ZEN_INFO("Power cycle mode enabled -- shutting down");
-
 		RequestExit(0);
 	}
 
@@ -707,10 +706,12 @@ ZenServer::Run()
 void
 ZenServer::RequestExit(int ExitCode)
 {
-	RequestApplicationExit(ExitCode);
-	if (m_Http)
+	if (RequestApplicationExit(ExitCode))
 	{
-		m_Http->RequestExit();
+		if (m_Http)
+		{
+			m_Http->RequestExit();
+		}
 	}
 }
 
@@ -779,7 +780,7 @@ ZenServer::EnsureIoRunner()
 }
 
 void
-ZenServer::EnqueueTimer()
+ZenServer::EnqueueProcessMonitorTimer()
 {
 	m_PidCheckTimer.expires_after(std::chrono::seconds(1));
 	m_PidCheckTimer.async_wait([this](const asio::error_code&) { CheckOwnerPid(); });
@@ -857,11 +858,10 @@ ZenServer::CheckSigInt()
 	EnqueueSigIntTimer();
 }
 
-void
-ZenServer::CheckOwnerPid()
+bool
+ZenServer::UpdateProcessMonitor()
 {
 	// Pick up any new "owner" processes
-
 	std::set<uint32_t> AddedPids;
 
 	for (auto& PidEntry : m_ServerEntry->SponsorPids)
@@ -874,21 +874,38 @@ ZenServer::CheckOwnerPid()
 				{
 					m_ProcessMonitor.AddPid(ThisPid);
 
-					ZEN_INFO("added process with pid #{} as a sponsor process", ThisPid);
+					ZEN_INFO("added process with pid {} as a sponsor process", ThisPid);
 				}
 			}
 		}
 	}
+	return m_ProcessMonitor.IsRunning();
+}
 
-	if (m_ProcessMonitor.IsRunning())
+void
+ZenServer::CheckOwnerPid()
+{
+	bool IsRunning = UpdateProcessMonitor();
+
+	if (IsRunning)
 	{
-		EnqueueTimer();
+		m_FoundNoActiveSponsors = false;
+		EnqueueProcessMonitorTimer();
 	}
 	else
 	{
-		ZEN_INFO(ZEN_APP_NAME " exiting since sponsor processes are all gone");
-
-		RequestExit(0);
+		// Delay exit one iteration to avoid race conditions where one process detaches
+		// and another attaches
+		if (m_FoundNoActiveSponsors)
+		{
+			ZEN_INFO(ZEN_APP_NAME " exiting since sponsor processes are all gone");
+			RequestExit(0);
+		}
+		else
+		{
+			m_FoundNoActiveSponsors = true;
+			EnqueueProcessMonitorTimer();
+		}
 	}
 }
 
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index cdd7c17a7..dd259f855 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -71,14 +71,14 @@ public:
 	void				  OnReady();
 
 	void EnsureIoRunner();
-	void EnqueueTimer();
+	void EnqueueProcessMonitorTimer();
 	void EnqueueStateMarkerTimer();
 	void EnqueueSigIntTimer();
 	void EnqueueStatsReportingTimer();
 	void CheckStateMarker();
 	void CheckSigInt();
 	void CheckOwnerPid();
-
+	bool UpdateProcessMonitor();
 	void ScrubStorage();
 	void Flush();
 
@@ -100,6 +100,7 @@ private:
 	asio::steady_timer				m_StatsReportingTimer{m_IoContext};
 	ProcessMonitor					m_ProcessMonitor;
 	NamedMutex						m_ServerMutex;
+	bool							m_FoundNoActiveSponsors = false;
 
 	enum ServerState
 	{