diff options
Diffstat (limited to 'src/zenutil/zenserverprocess.cpp')
| -rw-r--r-- | src/zenutil/zenserverprocess.cpp | 250 |
1 files changed, 228 insertions, 22 deletions
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index ebce9730e..2d4334ffa 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -15,6 +15,7 @@ #include <zencore/timer.h> #include <atomic> +#include <string> #include <gsl/gsl-lite.hpp> @@ -180,7 +181,7 @@ ZenServerState::Initialize() ThrowLastError("Could not map view of Zen server state"); } #else - int Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT | O_CLOEXEC, geteuid() == 0 ? 0766 : 0666); + int Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT, geteuid() == 0 ? 0766 : 0666); if (Fd < 0) { // Work around a potential issue if the service user is changed in certain configurations. @@ -190,7 +191,7 @@ ZenServerState::Initialize() // shared memory object and retry, we'll be able to get past shm_open() so long as we have // the appropriate permissions to create the shared memory object. shm_unlink("/UnrealEngineZen"); - Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT | O_CLOEXEC, geteuid() == 0 ? 0766 : 0666); + Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT, geteuid() == 0 ? 0766 : 0666); if (Fd < 0) { ThrowLastError("Could not open a shared memory object"); @@ -243,7 +244,7 @@ ZenServerState::InitializeReadOnly() ThrowLastError("Could not map view of Zen server state"); } #else - int Fd = shm_open("/UnrealEngineZen", O_RDONLY | O_CLOEXEC, 0666); + int Fd = shm_open("/UnrealEngineZen", O_RDONLY, 0666); if (Fd < 0) { return false; @@ -266,6 +267,8 @@ ZenServerState::InitializeReadOnly() ZenServerState::ZenServerEntry* ZenServerState::Lookup(int DesiredListenPort) const { + const uint32_t OurPid = GetCurrentProcessId(); + for (int i = 0; i < m_MaxEntryCount; ++i) { uint16_t EntryPort = m_Data[i].DesiredListenPort; @@ -273,6 +276,14 @@ ZenServerState::Lookup(int DesiredListenPort) const { if (DesiredListenPort == 0 || (EntryPort == DesiredListenPort)) { + // If the entry's PID matches our own but we haven't registered yet, + // this is a stale entry from a previous process incarnation (e.g. PID 1 + // reuse after unclean shutdown in k8s). Skip it. + if (m_Data[i].Pid == OurPid && m_OurEntry == nullptr) + { + continue; + } + std::error_code _; if (IsProcessRunning(m_Data[i].Pid, _)) { @@ -288,6 +299,8 @@ ZenServerState::Lookup(int DesiredListenPort) const ZenServerState::ZenServerEntry* ZenServerState::LookupByEffectivePort(int Port) const { + const uint32_t OurPid = GetCurrentProcessId(); + for (int i = 0; i < m_MaxEntryCount; ++i) { uint16_t EntryPort = m_Data[i].EffectiveListenPort; @@ -295,6 +308,11 @@ ZenServerState::LookupByEffectivePort(int Port) const { if (EntryPort == Port) { + if (m_Data[i].Pid == OurPid && m_OurEntry == nullptr) + { + continue; + } + std::error_code _; if (IsProcessRunning(m_Data[i].Pid, _)) { @@ -357,12 +375,26 @@ ZenServerState::Sweep() ZEN_ASSERT(m_IsReadOnly == false); + const uint32_t OurPid = GetCurrentProcessId(); + for (int i = 0; i < m_MaxEntryCount; ++i) { ZenServerEntry& Entry = m_Data[i]; if (Entry.DesiredListenPort) { + // If the entry's PID matches our own but we haven't registered yet, + // this is a stale entry from a previous process incarnation (e.g. PID 1 + // reuse after unclean shutdown in k8s). Reclaim it. + if (Entry.Pid == OurPid && m_OurEntry == nullptr) + { + ZEN_CONSOLE_DEBUG("Sweep - pid {} matches current process but no registration yet, reclaiming stale entry (port {})", + Entry.Pid.load(), + Entry.DesiredListenPort.load()); + Entry.Reset(); + continue; + } + std::error_code ErrorCode; if (Entry.Pid != 0 && IsProcessRunning(Entry.Pid, ErrorCode) == false) { @@ -619,7 +651,7 @@ ZenServerInstanceInfo::Create(const Oid& SessionId, const InstanceInfoData& Data ThrowLastError("Could not map instance info shared memory"); } #else - int Fd = shm_open(Name.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, 0666); + int Fd = shm_open(Name.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666); if (Fd < 0) { ThrowLastError("Could not create instance info shared memory"); @@ -686,7 +718,7 @@ ZenServerInstanceInfo::OpenReadOnly(const Oid& SessionId) return false; } #else - int Fd = shm_open(Name.c_str(), O_RDONLY | O_CLOEXEC, 0666); + int Fd = shm_open(Name.c_str(), O_RDONLY, 0666); if (Fd < 0) { return false; @@ -964,6 +996,7 @@ ZenServerInstance::Shutdown() ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid()); int ExitCode = m_Process.GetExitCode(); m_Process.Reset(); + m_ShutdownEvent.reset(); return ExitCode; } @@ -993,6 +1026,7 @@ ZenServerInstance::Shutdown() ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid()); int ExitCode = m_Process.GetExitCode(); m_Process.Reset(); + m_ShutdownEvent.reset(); return ExitCode; } else if (Ec) @@ -1020,6 +1054,7 @@ ZenServerInstance::Shutdown() int ExitCode = m_Process.GetExitCode(); ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid()); m_Process.Reset(); + m_ShutdownEvent.reset(); return ExitCode; } ZEN_DEBUG("Detached from zenserver process {} ({})", m_Name, m_Process.Pid()); @@ -1078,8 +1113,23 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, ChildEventName << "Zen_Child_" << ChildId; NamedEvent ChildEvent{ChildEventName}; + const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); + const std::filesystem::path Executable = + m_ServerExecutablePath.empty() ? (BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL) : m_ServerExecutablePath; + ExtendableStringBuilder<512> CommandLine; - CommandLine << "zenserver" ZEN_EXE_SUFFIX_LITERAL; // see CreateProc() call for actual binary path + { + const std::string ExeUtf8 = PathToUtf8(Executable); + constexpr AsciiSet QuoteChars = " \t\""; + if (AsciiSet::HasAny(ExeUtf8.c_str(), QuoteChars)) + { + CommandLine << '"' << ExeUtf8 << '"'; + } + else + { + CommandLine << ExeUtf8; + } + } if (m_ServerMode == ServerMode::kHubServer) { @@ -1092,6 +1142,11 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, CommandLine << " --child-id " << ChildEventName; + if (!m_EnableExecutionHistory) + { + CommandLine << " --enable-execution-history=false"; + } + if (!ServerArgs.empty()) { CommandLine << " " << ServerArgs; @@ -1106,10 +1161,6 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, { CreationFlags |= CreateProcOptions::Flag_NewConsole; } - - const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); - const std::filesystem::path Executable = - m_ServerExecutablePath.empty() ? (BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL) : m_ServerExecutablePath; const std::filesystem::path OutputPath = (OpenConsole || m_Env.IsPassthroughOutput()) ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); @@ -1371,18 +1422,31 @@ ZenServerInstance::OnServerReady() const ZenServerState::ZenServerEntry* Entry = nullptr; - if (m_BasePort) - { - Entry = State.Lookup(m_BasePort); - } - else + // The child process signals its ready event after writing its state entry, but under + // heavy instrumentation (e.g. sanitizers) the shared memory writes may not be immediately + // visible to this process. Retry briefly before giving up. + for (int Attempt = 0; Attempt < 10; ++Attempt) { - State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) { - if (InEntry.Pid == (uint32_t)m_Process.Pid()) - { - Entry = &InEntry; - } - }); + if (m_BasePort) + { + Entry = State.Lookup(m_BasePort); + } + else + { + State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) { + if (InEntry.Pid == (uint32_t)m_Process.Pid()) + { + Entry = &InEntry; + } + }); + } + + if (Entry) + { + break; + } + + Sleep(100); } if (!Entry) @@ -1419,7 +1483,7 @@ ZenServerInstance::SetDataDir(std::filesystem::path TestDir) } bool -ZenServerInstance::IsRunning() +ZenServerInstance::IsRunning() const { if (!m_Process.IsValid()) { @@ -1428,6 +1492,16 @@ ZenServerInstance::IsRunning() return m_Process.IsRunning(); } +void +ZenServerInstance::ResetDeadProcess() +{ + if (m_Process.IsValid() && !m_Process.IsRunning()) + { + m_Process.Reset(); + m_ShutdownEvent.reset(); + } +} + std::string ZenServerInstance::GetLogOutput() const { @@ -1553,4 +1627,136 @@ ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason) return true; } +std::optional<int> +StartupZenServer(LoggerRef LogRef, const StartupZenServerOptions& Options) +{ + ZEN_SCOPED_LOG(LogRef); + + // Check if a matching server is already running + { + ZenServerState State; + if (State.InitializeReadOnly()) + { + uint32_t RunningPid = 0; + uint16_t RunningEffectivePort = 0; + State.Snapshot([&, DesiredPort = Options.Port](const ZenServerState::ZenServerEntry& Entry) { + if (RunningPid == 0 && (DesiredPort == 0 || Entry.DesiredListenPort.load() == DesiredPort)) + { + RunningPid = Entry.Pid.load(); + RunningEffectivePort = Entry.EffectiveListenPort.load(); + } + }); + if (RunningPid != 0) + { + ZEN_INFO("Zen server already running at port {}, pid {}", RunningEffectivePort, RunningPid); + return std::nullopt; + } + } + } + + std::filesystem::path ProgramBaseDir = Options.ProgramBaseDir; + if (ProgramBaseDir.empty()) + { + ProgramBaseDir = GetRunningExecutablePath().parent_path(); + } + + ZenServerEnvironment ServerEnvironment; + ServerEnvironment.Initialize(ProgramBaseDir); + ZenServerInstance Server(ServerEnvironment, Options.Mode); + Server.SetEnableExecutionHistory(Options.EnableExecutionHistory); + + std::string ServerArguments(Options.ExtraArgs); + if ((Options.Port != 0) && (ServerArguments.find("--port") == std::string::npos)) + { + ServerArguments.append(fmt::format(" --port {}", Options.Port)); + } + Server.SpawnServer(ServerArguments, Options.OpenConsole, /*WaitTimeoutMs*/ 0); + + constexpr int Timeout = 10000; + + if (!Server.WaitUntilReady(Timeout)) + { + ZEN_WARN("{}", Server.GetLogOutput()); + if (Server.IsRunning()) + { + ZEN_WARN("Zen server launch failed (timed out), terminating"); + Server.Terminate(); + return 1; + } + int ExitCode = Server.Shutdown(); + ZEN_WARN("Zen server failed to get to a ready state and exited with return code {}", ExitCode); + return ExitCode != 0 ? ExitCode : 1; + } + + if (Options.ShowLog) + { + ZEN_INFO("{}", Server.GetLogOutput()); + } + return 0; +} + +bool +ShutdownZenServer(LoggerRef LogRef, + ZenServerState& State, + ZenServerState::ZenServerEntry* Entry, + const std::filesystem::path& ProgramBaseDir) +{ + ZEN_SCOPED_LOG(LogRef); + int EntryPort = (int)Entry->DesiredListenPort.load(); + const uint32_t ServerProcessPid = Entry->Pid.load(); + try + { + ZenServerEnvironment ServerEnvironment; + ServerEnvironment.Initialize(ProgramBaseDir); + ZenServerInstance Server(ServerEnvironment); + Server.AttachToRunningServer(EntryPort); + + ZEN_INFO("attached to server on port {} (pid {}), requesting shutdown", EntryPort, ServerProcessPid); + + std::error_code Ec; + if (Server.SignalShutdown(Ec) && !Ec) + { + Stopwatch Timer; + while (Timer.GetElapsedTimeMs() < 10000) + { + if (Server.WaitUntilExited(100, Ec) && !Ec) + { + ZEN_INFO("shutdown complete"); + return true; + } + else if (Ec) + { + ZEN_WARN("Waiting for server on port {} (pid {}) failed. Reason: '{}'", EntryPort, ServerProcessPid, Ec.message()); + } + } + } + else if (Ec) + { + ZEN_WARN("Requesting shutdown of server on port {} failed. Reason: '{}'", EntryPort, Ec.message()); + } + } + catch (const std::exception& Ex) + { + ZEN_DEBUG("Exception caught when requesting shutdown: {}", Ex.what()); + } + + ZEN_INFO("Requesting detached shutdown of server on port {}", EntryPort); + Entry->SignalShutdownRequest(); + + Stopwatch Timer; + while (Timer.GetElapsedTimeMs() < 10000) + { + State.Sweep(); + Entry = State.Lookup(EntryPort); + if (Entry == nullptr || Entry->Pid.load() != ServerProcessPid) + { + ZEN_INFO("Shutdown complete"); + return true; + } + Sleep(100); + } + + return false; +} + } // namespace zen |