diff options
Diffstat (limited to 'src/zenserver/hub/storageserverinstance.cpp')
| -rw-r--r-- | src/zenserver/hub/storageserverinstance.cpp | 57 |
1 files changed, 54 insertions, 3 deletions
diff --git a/src/zenserver/hub/storageserverinstance.cpp b/src/zenserver/hub/storageserverinstance.cpp index 7933cfa70..0a9efcc44 100644 --- a/src/zenserver/hub/storageserverinstance.cpp +++ b/src/zenserver/hub/storageserverinstance.cpp @@ -28,6 +28,7 @@ void StorageServerInstance::SpawnServerProcess() { ZEN_ASSERT_FORMAT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId); + m_ServerInstance.ResetDeadProcess(); m_ServerInstance.SetServerExecutablePath(GetRunningExecutablePath()); m_ServerInstance.SetDataDir(m_BaseDir); @@ -77,6 +78,12 @@ StorageServerInstance::ProvisionLocked() return; } + if (m_State.load() == HubInstanceState::Crashed) + { + ZEN_WARN("Storage server instance for module '{}' is in crashed state; re-provisioning from scratch", m_ModuleId); + m_State = HubInstanceState::Unprovisioned; + } + if (m_State.load() == HubInstanceState::Hibernated) { if (WakeLocked()) @@ -105,18 +112,23 @@ StorageServerInstance::ProvisionLocked() void StorageServerInstance::DeprovisionLocked() { - if (m_State.load() != HubInstanceState::Provisioned) + const HubInstanceState CurrentState = m_State.load(); + if (CurrentState != HubInstanceState::Provisioned && CurrentState != HubInstanceState::Crashed) { ZEN_WARN("Attempted to deprovision storage server instance for module '{}' which is not provisioned (state: '{}')", m_ModuleId, - ToString(m_State.load())); + ToString(CurrentState)); return; } ZEN_INFO("Deprovisioning storage server instance for module '{}'", m_ModuleId); m_State = HubInstanceState::Deprovisioning; - m_ServerInstance.Shutdown(); + if (CurrentState == HubInstanceState::Provisioned) + { + m_ServerInstance.Shutdown(); // Graceful; process may still be running + } + // Crashed: process already dead; skip Shutdown Dehydrate(); @@ -187,6 +199,29 @@ StorageServerInstance::WakeLocked() } } +bool +StorageServerInstance::RecoverCrashedLocked() +{ + ZEN_ASSERT(m_State.load() == HubInstanceState::Provisioned); + ZEN_ASSERT(!m_ServerInstance.IsRunning()); + + ZEN_WARN("Storage server instance for module '{}' has crashed; attempting in-place recovery", m_ModuleId); + m_State = HubInstanceState::Recovering; + try + { + SpawnServerProcess(); + m_State = HubInstanceState::Provisioned; + ZEN_INFO("Storage server instance for module '{}' recovered successfully", m_ModuleId); + return true; + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed to restart module '{}': {}", m_ModuleId, Ex.what()); + m_State = HubInstanceState::Crashed; + return false; + } +} + void StorageServerInstance::Hydrate() { @@ -303,6 +338,15 @@ StorageServerInstance::UpdateMetricsLocked() // TODO: Resource metrics... } +#if ZEN_WITH_TESTS +void +StorageServerInstance::SharedLockedPtr::TerminateForTesting() const +{ + ZEN_ASSERT(m_Instance != nullptr); + m_Instance->m_ServerInstance.Terminate(); +} +#endif + StorageServerInstance::ExclusiveLockedPtr::ExclusiveLockedPtr() : m_Lock(nullptr), m_Instance(nullptr) { } @@ -402,4 +446,11 @@ StorageServerInstance::ExclusiveLockedPtr::Wake() return m_Instance->WakeLocked(); } +bool +StorageServerInstance::ExclusiveLockedPtr::RecoverFromCrash() +{ + ZEN_ASSERT(m_Instance != nullptr); + return m_Instance->RecoverCrashedLocked(); +} + } // namespace zen |