zen hub command (#877)

- Feature: Added `zen hub` command for managing a hub server and its provisioned module instances:
  - `zen hub up` - Start a hub server (equivalent to `zen up` in hub mode)
  - `zen hub down` - Shut down a hub server
  - `zen hub provision <moduleid>` - Provision a storage server instance for a module
  - `zen hub deprovision <moduleid>` - Deprovision a storage server instance
  - `zen hub hibernate <moduleid>` - Hibernate a provisioned instance (shut down, data preserved)
  - `zen hub wake <moduleid>` - Wake a hibernated instance
  - `zen hub status [moduleid]` - Show state of all instances or a specific module
- Feature: Added new hub HTTP endpoints for instance lifecycle management:
  - `POST /hub/modules/{moduleid}/hibernate` - Hibernate the instance for the given module
  - `POST /hub/modules/{moduleid}/wake` - Wake a hibernated instance for the given module
- Improvement: `zen up` refactored to use shared `StartupZenServer`/`ShutdownZenServer` helpers (also used by `zen hub up`/`zen hub down`)
- Bugfix: Fixed shutdown event not being cleared after the server process exits in `ZenServerInstance::Shutdown()`, which could cause stale state on reuse

1 files changed, 135 insertions, 0 deletions

diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 8eaf2cf5b..a2ab4c291 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -15,6 +15,7 @@
 #include <zencore/timer.h>
 
 #include <atomic>
+#include <string>
 
 #include <gsl/gsl-lite.hpp>
 
@@ -964,6 +965,7 @@ ZenServerInstance::Shutdown()
 					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
 					int ExitCode = m_Process.GetExitCode();
 					m_Process.Reset();
+					m_ShutdownEvent.reset();
 					return ExitCode;
 				}
 
@@ -993,6 +995,7 @@ ZenServerInstance::Shutdown()
 					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
 					int ExitCode = m_Process.GetExitCode();
 					m_Process.Reset();
+					m_ShutdownEvent.reset();
 					return ExitCode;
 				}
 				else if (Ec)
@@ -1020,6 +1023,7 @@ ZenServerInstance::Shutdown()
 				int ExitCode = m_Process.GetExitCode();
 				ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
 				m_Process.Reset();
+				m_ShutdownEvent.reset();
 				return ExitCode;
 			}
 			ZEN_DEBUG("Detached from zenserver process {} ({})", m_Name, m_Process.Pid());
@@ -1553,4 +1557,135 @@ ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason)
 	return true;
 }
 
+std::optional<int>
+StartupZenServer(LoggerRef LogRef, const StartupZenServerOptions& Options)
+{
+	auto Log = [&LogRef]() { return LogRef; };
+
+	// Check if a matching server is already running
+	{
+		ZenServerState State;
+		if (State.InitializeReadOnly())
+		{
+			uint32_t RunningPid			  = 0;
+			uint16_t RunningEffectivePort = 0;
+			State.Snapshot([&, DesiredPort = Options.Port](const ZenServerState::ZenServerEntry& Entry) {
+				if (RunningPid == 0 && (DesiredPort == 0 || Entry.DesiredListenPort.load() == DesiredPort))
+				{
+					RunningPid			 = Entry.Pid.load();
+					RunningEffectivePort = Entry.EffectiveListenPort.load();
+				}
+			});
+			if (RunningPid != 0)
+			{
+				ZEN_INFO("Zen server already running at port {}, pid {}", RunningEffectivePort, RunningPid);
+				return std::nullopt;
+			}
+		}
+	}
+
+	std::filesystem::path ProgramBaseDir = Options.ProgramBaseDir;
+	if (ProgramBaseDir.empty())
+	{
+		ProgramBaseDir = GetRunningExecutablePath().parent_path();
+	}
+
+	ZenServerEnvironment ServerEnvironment;
+	ServerEnvironment.Initialize(ProgramBaseDir);
+	ZenServerInstance Server(ServerEnvironment, Options.Mode);
+
+	std::string ServerArguments(Options.ExtraArgs);
+	if ((Options.Port != 0) && (ServerArguments.find("--port") == std::string::npos))
+	{
+		ServerArguments.append(fmt::format(" --port {}", Options.Port));
+	}
+	Server.SpawnServer(ServerArguments, Options.OpenConsole, /*WaitTimeoutMs*/ 0);
+
+	constexpr int Timeout = 10000;
+
+	if (!Server.WaitUntilReady(Timeout))
+	{
+		ZEN_WARN("{}", Server.GetLogOutput());
+		if (Server.IsRunning())
+		{
+			ZEN_WARN("Zen server launch failed (timed out), terminating");
+			Server.Terminate();
+			return 1;
+		}
+		int ExitCode = Server.Shutdown();
+		ZEN_WARN("Zen server failed to get to a ready state and exited with return code {}", ExitCode);
+		return ExitCode != 0 ? ExitCode : 1;
+	}
+
+	if (Options.ShowLog)
+	{
+		ZEN_INFO("{}", Server.GetLogOutput());
+	}
+	return 0;
+}
+
+bool
+ShutdownZenServer(LoggerRef						  LogRef,
+				  ZenServerState&				  State,
+				  ZenServerState::ZenServerEntry* Entry,
+				  const std::filesystem::path&	  ProgramBaseDir)
+{
+	auto		   Log				= [&LogRef]() { return LogRef; };
+	int			   EntryPort		= (int)Entry->DesiredListenPort.load();
+	const uint32_t ServerProcessPid = Entry->Pid.load();
+	try
+	{
+		ZenServerEnvironment ServerEnvironment;
+		ServerEnvironment.Initialize(ProgramBaseDir);
+		ZenServerInstance Server(ServerEnvironment);
+		Server.AttachToRunningServer(EntryPort);
+
+		ZEN_INFO("attached to server on port {} (pid {}), requesting shutdown", EntryPort, ServerProcessPid);
+
+		std::error_code Ec;
+		if (Server.SignalShutdown(Ec) && !Ec)
+		{
+			Stopwatch Timer;
+			while (Timer.GetElapsedTimeMs() < 10000)
+			{
+				if (Server.WaitUntilExited(100, Ec) && !Ec)
+				{
+					ZEN_INFO("shutdown complete");
+					return true;
+				}
+				else if (Ec)
+				{
+					ZEN_WARN("Waiting for server on port {} (pid {}) failed. Reason: '{}'", EntryPort, ServerProcessPid, Ec.message());
+				}
+			}
+		}
+		else if (Ec)
+		{
+			ZEN_WARN("Requesting shutdown of server on port {} failed. Reason: '{}'", EntryPort, Ec.message());
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_DEBUG("Exception caught when requesting shutdown: {}", Ex.what());
+	}
+
+	ZEN_INFO("Requesting detached shutdown of server on port {}", EntryPort);
+	Entry->SignalShutdownRequest();
+
+	Stopwatch Timer;
+	while (Timer.GetElapsedTimeMs() < 10000)
+	{
+		State.Sweep();
+		Entry = State.Lookup(EntryPort);
+		if (Entry == nullptr || Entry->Pid.load() != ServerProcessPid)
+		{
+			ZEN_INFO("Shutdown complete");
+			return true;
+		}
+		Sleep(100);
+	}
+
+	return false;
+}
+
 }  // namespace zen