zen hub command (#877)

- Feature: Added `zen hub` command for managing a hub server and its provisioned module instances:
  - `zen hub up` - Start a hub server (equivalent to `zen up` in hub mode)
  - `zen hub down` - Shut down a hub server
  - `zen hub provision <moduleid>` - Provision a storage server instance for a module
  - `zen hub deprovision <moduleid>` - Deprovision a storage server instance
  - `zen hub hibernate <moduleid>` - Hibernate a provisioned instance (shut down, data preserved)
  - `zen hub wake <moduleid>` - Wake a hibernated instance
  - `zen hub status [moduleid]` - Show state of all instances or a specific module
- Feature: Added new hub HTTP endpoints for instance lifecycle management:
  - `POST /hub/modules/{moduleid}/hibernate` - Hibernate the instance for the given module
  - `POST /hub/modules/{moduleid}/wake` - Wake a hibernated instance for the given module
- Improvement: `zen up` refactored to use shared `StartupZenServer`/`ShutdownZenServer` helpers (also used by `zen hub up`/`zen hub down`)
- Bugfix: Fixed shutdown event not being cleared after the server process exits in `ZenServerInstance::Shutdown()`, which could cause stale state on reuse

1 files changed, 19 insertions, 156 deletions

diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp
index db2c77b6b..809a41bb6 100644
--- a/src/zen/cmds/up_cmd.cpp
+++ b/src/zen/cmds/up_cmd.cpp
@@ -8,25 +8,21 @@
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/process.h>
-#include <zencore/timer.h>
 #include <zenutil/zenserverprocess.h>
 
-#include <memory>
-
 namespace zen {
 
 UpCommand::UpCommand()
 {
 	m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("0"), "<hostport>");
 	m_Options.add_option("", "b", "base-dir", "Parent folder of server executable", cxxopts::value(m_ProgramBaseDir), "<directory>");
-	m_Options
-		.add_option("", "c", "show-console", "Open a console window for the zenserver process", cxxopts::value(m_ShowConsole), "<console>");
+	m_Options.add_option("", "c", "show-console", "Open a console window for the zenserver process", cxxopts::value(m_ShowConsole), "");
 	m_Options.add_option("",
 						 "l",
 						 "show-log",
 						 "Show the output log of the zenserver process after successful start",
 						 cxxopts::value(m_ShowLog),
-						 "<showlog>");
+						 "");
 }
 
 UpCommand::~UpCommand() = default;
@@ -34,10 +30,6 @@ UpCommand::~UpCommand() = default;
 void
 UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 {
-	using namespace std::literals;
-
-	ZEN_UNUSED(GlobalOptions);
-
 	if (!ParseOptions(argc, argv))
 	{
 		return;
@@ -45,91 +37,26 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	if (m_ShowConsole && m_ShowLog)
 	{
-		throw OptionParseException("'--show-console' conficts with '--show-log'", m_Options.help());
-	}
-
-	{
-		ZenServerState State;
-		if (State.InitializeReadOnly())
-		{
-			struct EntryInfo
-			{
-				uint32_t Pid		   = 0;
-				uint16_t DesiredPort   = 0;
-				uint16_t EffectivePort = 0;
-			};
-			std::vector<EntryInfo> RunningEntries;
-			State.Snapshot([&RunningEntries, DesiredPort = this->m_Port](const zen::ZenServerState::ZenServerEntry& Entry) {
-				if (DesiredPort == 0 || Entry.DesiredListenPort.load() == DesiredPort)
-				{
-					RunningEntries.push_back(EntryInfo{.Pid			  = Entry.Pid.load(),
-													   .DesiredPort	  = Entry.DesiredListenPort.load(),
-													   .EffectivePort = Entry.EffectiveListenPort.load()});
-				}
-			});
-			if (RunningEntries.size() > 0)
-			{
-				ZEN_CONSOLE("Zen server already running with base port {}. First instance at port {}, pid {}",
-							RunningEntries[0].DesiredPort,
-							RunningEntries[0].EffectivePort,
-							RunningEntries[0].Pid);
-				return;
-			}
-		}
+		throw OptionParseException("'--show-console' conflicts with '--show-log'", m_Options.help());
 	}
 
-	if (m_ProgramBaseDir.empty())
+	std::optional<int> StartResult = StartupZenServer(ConsoleLog(),
+													  {.ProgramBaseDir = m_ProgramBaseDir,
+													   .Port		   = m_Port,
+													   .OpenConsole	   = m_ShowConsole,
+													   .ShowLog		   = m_ShowLog,
+													   .ExtraArgs	   = GlobalOptions.PassthroughCommandLine});
+	if (!StartResult.has_value())
 	{
-		std::filesystem::path ExePath = zen::GetRunningExecutablePath();
-		m_ProgramBaseDir			  = ExePath.parent_path();
+		ZEN_CONSOLE("Zen server already running");
+		return;
 	}
-	ZenServerEnvironment ServerEnvironment;
-	ServerEnvironment.Initialize(m_ProgramBaseDir);
-	ZenServerInstance Server(ServerEnvironment);
-	std::string		  ServerArguments = GlobalOptions.PassthroughCommandLine;
-	if ((m_Port != 0) && (ServerArguments.find("--port"sv) == std::string::npos))
+	if (*StartResult != 0)
 	{
-		ServerArguments.append(fmt::format(" --port {}", m_Port));
+		throw ErrorWithReturnCode("Zen server failed to start", *StartResult);
 	}
-	Server.SpawnServer(ServerArguments, m_ShowConsole, /*WaitTimeoutMs*/ 0);
 
-	int Timeout = 10000;
-
-	if (!Server.WaitUntilReady(Timeout))
-	{
-		if (Server.IsRunning())
-		{
-			ZEN_CONSOLE_WARN("Zen server launch failed (timed out), terminating");
-			Server.Terminate();
-			if (!m_ShowConsole)
-			{
-				ZEN_CONSOLE("{}", Server.GetLogOutput());
-			}
-			throw std::runtime_error("Zen server launch failed (timed out), launched process was terminated");
-		}
-		int ServerExitCode = Server.Shutdown();
-		if (!m_ShowConsole)
-		{
-			ZEN_CONSOLE("{}", Server.GetLogOutput());
-		}
-		if (ServerExitCode != 0)
-		{
-			throw ErrorWithReturnCode(
-				fmt::format("Zen server failed to get to a ready state and exited with return code {}", ServerExitCode),
-				ServerExitCode);
-		}
-	}
-	else
-	{
-		if (m_ShowLog)
-		{
-			ZEN_CONSOLE("{}", Server.GetLogOutput());
-		}
-		else
-		{
-			ZEN_CONSOLE("Zen server up");
-		}
-	}
+	ZEN_CONSOLE("Zen server up");
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -211,70 +138,6 @@ DownCommand::DownCommand()
 
 DownCommand::~DownCommand() = default;
 
-bool
-DownCommand::ShutdownEntry(ZenServerState& Instance, ZenServerState::ZenServerEntry* Entry)
-{
-	int			   EntryPort		= (int)Entry->DesiredListenPort.load();
-	const uint32_t ServerProcessPid = Entry->Pid.load();
-	try
-	{
-		ZenServerEnvironment ServerEnvironment;
-		ServerEnvironment.Initialize(m_ProgramBaseDir);
-		ZenServerInstance Server(ServerEnvironment);
-		Server.AttachToRunningServer(EntryPort);
-
-		ZEN_CONSOLE("attached to server on port {} (pid {}), requesting shutdown", EntryPort, ServerProcessPid);
-
-		std::error_code Ec;
-		if (Server.SignalShutdown(Ec) && !Ec)
-		{
-			Stopwatch Timer;
-			while (Timer.GetElapsedTimeMs() < 10000)
-			{
-				if (Server.WaitUntilExited(100, Ec) && !Ec)
-				{
-					ZEN_CONSOLE("shutdown complete");
-					return true;
-				}
-				else if (Ec)
-				{
-					ZEN_CONSOLE("Waiting for server on port {} (pid {}) failed. Reason: '{}'", EntryPort, ServerProcessPid, Ec.message());
-				}
-			}
-		}
-		else if (Ec)
-		{
-			ZEN_CONSOLE_WARN("Requesting shutdown of server on port {} failed. Reason: '{}'", EntryPort, Ec.message());
-		}
-	}
-	catch (const std::exception& Ex)
-	{
-		ZEN_DEBUG("Exception caught when requesting shutdown: {}", Ex.what());
-	}
-
-	// Since we cannot obtain a handle to the process we are unable to block on the process
-	// handle to determine when the server has shut down. Thus we signal that we would like
-	// a shutdown via the shutdown flag and the check if the entry is still running.
-
-	ZEN_CONSOLE("Requesting detached shutdown of server on port {}", EntryPort);
-	Entry->SignalShutdownRequest();
-
-	Stopwatch Timer;
-	while (Timer.GetElapsedTimeMs() < 10000)
-	{
-		Instance.Sweep();
-		Entry = Instance.Lookup(EntryPort);
-		if (Entry == nullptr || Entry->Pid.load() != ServerProcessPid)
-		{
-			ZEN_CONSOLE("Shutdown complete");
-			return true;
-		}
-		Sleep(100);
-	}
-
-	return false;
-}
-
 void
 DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 {
@@ -325,7 +188,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			ZenServerState::ZenServerEntry* Entry = Instance.Lookup(Info.Port);
 			if (Entry && Entry->Pid.load() == Info.Pid)
 			{
-				if (!ShutdownEntry(Instance, Entry))
+				if (!ShutdownZenServer(ConsoleLog(), Instance, Entry, m_ProgramBaseDir))
 				{
 					ZEN_CONSOLE_WARN("Failed to shutdown server on port {} (pid {})", Info.Port, Info.Pid);
 					++FailCount;
@@ -370,7 +233,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	if (Entry)
 	{
-		if (ShutdownEntry(Instance, Entry))
+		if (ShutdownZenServer(ConsoleLog(), Instance, Entry, m_ProgramBaseDir))
 		{
 			return;
 		}
@@ -381,7 +244,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		// Try to find the running executable by path name
 		std::filesystem::path ServerExePath = m_ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
 		ProcessHandle		  RunningProcess;
-		if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec, /*IncludeSelf*/ false)
+		if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess, /*IncludeSelf*/ false); !Ec)
 		{
 			ZEN_CONSOLE_WARN("Attempting hard terminate of zen process with pid ({})", RunningProcess.Pid());
 			if (RunningProcess.Terminate(0))
@@ -399,7 +262,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	else if (Entry)
 	{
 		throw std::runtime_error(
-			fmt::format("Failed to shutdown of server on port {}, use --force to hard terminate process", Entry->DesiredListenPort.load()));
+			fmt::format("Failed to shut down server on port {}, use --force to hard terminate process", Entry->DesiredListenPort.load()));
 	}
 
 	ZEN_CONSOLE("No zen server to bring down");