refactor hub notifications (#888)

* refactor hub callbacks
* improve http responses

1 files changed, 50 insertions, 29 deletions

diff --git a/src/zenserver/hub/storageserverinstance.cpp b/src/zenserver/hub/storageserverinstance.cpp
index 0a9efcc44..99f0c29f3 100644
--- a/src/zenserver/hub/storageserverinstance.cpp
+++ b/src/zenserver/hub/storageserverinstance.cpp
@@ -69,13 +69,13 @@ StorageServerInstance::GetProcessMetrics(ProcessMetrics& OutMetrics) const
 	OutMetrics.PeakPagefileUsage  = m_PeakPagefileUsage.load();
 }
 
-void
+bool
 StorageServerInstance::ProvisionLocked()
 {
 	if (m_State.load() == HubInstanceState::Provisioned)
 	{
 		ZEN_WARN("Storage server instance for module '{}' is already provisioned", m_ModuleId);
-		return;
+		return false;
 	}
 
 	if (m_State.load() == HubInstanceState::Crashed)
@@ -86,39 +86,45 @@ StorageServerInstance::ProvisionLocked()
 
 	if (m_State.load() == HubInstanceState::Hibernated)
 	{
-		if (WakeLocked())
-		{
-			return;
-		}
-		// Wake failed; proceed with a fresh provision (discards hibernated data)
-		m_State = HubInstanceState::Unprovisioned;
+		return WakeLocked();
 	}
-	else if (m_State.load() != HubInstanceState::Unprovisioned)
+
+	if (m_State.load() != HubInstanceState::Unprovisioned)
 	{
 		ZEN_WARN("Storage server instance for module '{}' is in unexpected state '{}', cannot provision",
 				 m_ModuleId,
 				 ToString(m_State.load()));
-		return;
+		return false;
 	}
 
 	ZEN_INFO("Provisioning storage server instance for module '{}', at '{}'", m_ModuleId, m_BaseDir);
 
 	m_State = HubInstanceState::Provisioning;
-	Hydrate();
-	SpawnServerProcess();
-	m_State = HubInstanceState::Provisioned;
+	try
+	{
+		Hydrate();
+		SpawnServerProcess();
+		m_State = HubInstanceState::Provisioned;
+		return true;
+	}
+	catch (...)
+	{
+		m_State = HubInstanceState::Unprovisioned;
+		throw;
+	}
 }
 
-void
+bool
 StorageServerInstance::DeprovisionLocked()
 {
 	const HubInstanceState CurrentState = m_State.load();
-	if (CurrentState != HubInstanceState::Provisioned && CurrentState != HubInstanceState::Crashed)
+	if (CurrentState != HubInstanceState::Provisioned && CurrentState != HubInstanceState::Crashed &&
+		CurrentState != HubInstanceState::Hibernated)
 	{
 		ZEN_WARN("Attempted to deprovision storage server instance for module '{}' which is not provisioned (state: '{}')",
 				 m_ModuleId,
 				 ToString(CurrentState));
-		return;
+		return false;
 	}
 
 	ZEN_INFO("Deprovisioning storage server instance for module '{}'", m_ModuleId);
@@ -126,13 +132,30 @@ StorageServerInstance::DeprovisionLocked()
 	m_State = HubInstanceState::Deprovisioning;
 	if (CurrentState == HubInstanceState::Provisioned)
 	{
-		m_ServerInstance.Shutdown();  // Graceful; process may still be running
+		try
+		{
+			m_ServerInstance.Shutdown();
+		}
+		catch (...)
+		{
+			m_State = HubInstanceState::Provisioned;  // Shutdown failed; process may still be running
+			throw;
+		}
 	}
-	// Crashed: process already dead; skip Shutdown
+	// Crashed or Hibernated: process already dead; skip Shutdown
 
-	Dehydrate();
+	try
+	{
+		Dehydrate();
+	}
+	catch (...)
+	{
+		m_State = HubInstanceState::Crashed;  // Dehydrate failed; process is already dead
+		throw;
+	}
 
 	m_State = HubInstanceState::Unprovisioned;
+	return true;
 }
 
 bool
@@ -161,11 +184,10 @@ StorageServerInstance::HibernateLocked()
 		m_State = HubInstanceState::Hibernated;
 		return true;
 	}
-	catch (const std::exception& Ex)
+	catch (...)
 	{
-		ZEN_ERROR("Failed to hibernate storage server instance for module '{}': {}", m_ModuleId, Ex.what());
 		m_State = HubInstanceState::Provisioned;  // Shutdown failed; instance is still running
-		return false;
+		throw;
 	}
 }
 
@@ -191,11 +213,10 @@ StorageServerInstance::WakeLocked()
 		m_State = HubInstanceState::Provisioned;
 		return true;
 	}
-	catch (const std::exception& Ex)
+	catch (...)
 	{
-		ZEN_ERROR("Failed to wake storage server instance for module '{}': {}", m_ModuleId, Ex.what());
 		m_State = HubInstanceState::Hibernated;
-		return false;
+		throw;
 	}
 }
 
@@ -418,18 +439,18 @@ StorageServerInstance::ExclusiveLockedPtr::IsRunning() const
 	return m_Instance->m_State.load() == HubInstanceState::Provisioned && m_Instance->m_ServerInstance.IsRunning();
 }
 
-void
+bool
 StorageServerInstance::ExclusiveLockedPtr::Provision()
 {
 	ZEN_ASSERT(m_Instance != nullptr);
-	m_Instance->ProvisionLocked();
+	return m_Instance->ProvisionLocked();
 }
 
-void
+bool
 StorageServerInstance::ExclusiveLockedPtr::Deprovision()
 {
 	ZEN_ASSERT(m_Instance != nullptr);
-	m_Instance->DeprovisionLocked();
+	return m_Instance->DeprovisionLocked();
 }
 
 bool