// Copyright Epic Games, Inc. All Rights Reserved. #include "zenutil/zenserverprocess.h" #include #include #include #include #include #include #include #include #include #include #include #include #if ZEN_PLATFORM_WINDOWS # include #else # include # include # include # include #endif ////////////////////////////////////////////////////////////////////////// namespace zen { namespace zenutil { #if ZEN_PLATFORM_WINDOWS class SecurityAttributes { public: inline SECURITY_ATTRIBUTES* Attributes() { return &m_Attributes; } protected: SECURITY_ATTRIBUTES m_Attributes{}; SECURITY_DESCRIPTOR m_Sd{}; }; // Security attributes which allows any user access class AnyUserSecurityAttributes : public SecurityAttributes { public: AnyUserSecurityAttributes() { m_Attributes.nLength = sizeof m_Attributes; m_Attributes.bInheritHandle = false; // Disable inheritance const BOOL Success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION); if (Success) { if (!SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE)) { ThrowLastError("SetSecurityDescriptorDacl failed"); } m_Attributes.lpSecurityDescriptor = &m_Sd; } } }; #endif // ZEN_PLATFORM_WINDOWS } // namespace zenutil ////////////////////////////////////////////////////////////////////////// ZenServerState::ZenServerState() { } ZenServerState::~ZenServerState() { if (m_OurEntry) { // Clean up our entry now that we're leaving m_OurEntry->Reset(); m_OurEntry = nullptr; } #if ZEN_PLATFORM_WINDOWS if (m_Data) { UnmapViewOfFile(m_Data); } if (m_hMapFile) { CloseHandle(m_hMapFile); } #else if (m_Data != nullptr) { munmap(m_Data, m_MaxEntryCount * sizeof(ZenServerEntry)); } int Fd = int(intptr_t(m_hMapFile)); close(Fd); #endif m_Data = nullptr; } void ZenServerState::Initialize() { size_t MapSize = m_MaxEntryCount * sizeof(ZenServerEntry); #if ZEN_PLATFORM_WINDOWS // TODO: there's a small chance of a race here, this logic could be tightened up with a mutex to // ensure only a single process at a time creates the mapping // TODO: the fallback to Local instead of Global has a flaw where if you start a non-elevated instance // first then start an elevated instance second you'll have the first instance with a local // mapping and the second instance with a global mapping. This kind of elevated/non-elevated // shouldn't be common, but handling for it should be improved in the future. HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap"); if (hMap == NULL) { hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Local\\ZenMap"); } if (hMap == NULL) { // Security attributes to enable any user to access state zenutil::AnyUserSecurityAttributes Attrs; hMap = CreateFileMapping(INVALID_HANDLE_VALUE, // use paging file Attrs.Attributes(), // allow anyone to access PAGE_READWRITE, // read/write access 0, // maximum object size (high-order DWORD) DWORD(MapSize), // maximum object size (low-order DWORD) L"Global\\ZenMap"); // name of mapping object if (hMap == NULL) { hMap = CreateFileMapping(INVALID_HANDLE_VALUE, // use paging file Attrs.Attributes(), // allow anyone to access PAGE_READWRITE, // read/write access 0, // maximum object size (high-order DWORD) m_MaxEntryCount * sizeof(ZenServerEntry), // maximum object size (low-order DWORD) L"Local\\ZenMap"); // name of mapping object } if (hMap == NULL) { ThrowLastError("Could not open or create file mapping object for Zen server state"); } } void* pBuf = MapViewOfFile(hMap, // handle to map object FILE_MAP_ALL_ACCESS, // read/write permission 0, // offset high 0, // offset low DWORD(MapSize)); if (pBuf == NULL) { ThrowLastError("Could not map view of Zen server state"); } #else int Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT | O_CLOEXEC, 0666); if (Fd < 0) { ThrowLastError("Could not open a shared memory object"); } fchmod(Fd, 0666); void* hMap = (void*)intptr_t(Fd); int Result = ftruncate(Fd, MapSize); ZEN_UNUSED(Result); void* pBuf = mmap(nullptr, MapSize, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0); if (pBuf == MAP_FAILED) { close(Fd); ThrowLastError("Could not map view of Zen server state"); } #endif m_hMapFile = hMap; m_Data = reinterpret_cast(pBuf); m_IsReadOnly = false; } bool ZenServerState::InitializeReadOnly() { size_t MapSize = m_MaxEntryCount * sizeof(ZenServerEntry); #if ZEN_PLATFORM_WINDOWS HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap"); if (hMap == NULL) { hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Local\\ZenMap"); } if (hMap == NULL) { return false; } void* pBuf = MapViewOfFile(hMap, // handle to map object FILE_MAP_READ, // read permission 0, // offset high 0, // offset low MapSize); if (pBuf == NULL) { ThrowLastError("Could not map view of Zen server state"); } #else int Fd = shm_open("/UnrealEngineZen", O_RDONLY | O_CLOEXEC, 0666); if (Fd < 0) { return false; } void* hMap = (void*)intptr_t(Fd); void* pBuf = mmap(nullptr, MapSize, PROT_READ, MAP_SHARED, Fd, 0); if (pBuf == MAP_FAILED) { ThrowLastError("Could not map read-only view of Zen server state"); } #endif m_hMapFile = hMap; m_Data = reinterpret_cast(pBuf); return true; } ZenServerState::ZenServerEntry* ZenServerState::Lookup(int DesiredListenPort) const { for (int i = 0; i < m_MaxEntryCount; ++i) { uint16_t EntryPort = m_Data[i].DesiredListenPort; if (EntryPort != 0) { if (DesiredListenPort == 0 || (EntryPort == DesiredListenPort)) { if (IsProcessRunning(m_Data[i].Pid)) { return &m_Data[i]; } } } } return nullptr; } ZenServerState::ZenServerEntry* ZenServerState::LookupByEffectivePort(int Port) const { for (int i = 0; i < m_MaxEntryCount; ++i) { uint16_t EntryPort = m_Data[i].EffectiveListenPort; if (EntryPort != 0) { if (EntryPort == Port) { if (IsProcessRunning(m_Data[i].Pid)) { return &m_Data[i]; } } } } return nullptr; } ZenServerState::ZenServerEntry* ZenServerState::Register(int DesiredListenPort) { if (m_Data == nullptr) { return nullptr; } // Allocate an entry int Pid = GetCurrentProcessId(); for (int i = 0; i < m_MaxEntryCount; ++i) { ZenServerEntry& Entry = m_Data[i]; if (Entry.DesiredListenPort.load(std::memory_order_relaxed) == 0) { uint16_t Expected = 0; if (Entry.DesiredListenPort.compare_exchange_strong(Expected, uint16_t(DesiredListenPort))) { // Successfully allocated entry m_OurEntry = &Entry; Entry.Pid = Pid; Entry.EffectiveListenPort = 0; Entry.Flags = 0; const Oid SesId = GetSessionId(); memcpy(Entry.SessionId, &SesId, sizeof SesId); return &Entry; } } } return nullptr; } void ZenServerState::Sweep() { if (m_Data == nullptr) { return; } ZEN_ASSERT(m_IsReadOnly == false); for (int i = 0; i < m_MaxEntryCount; ++i) { ZenServerEntry& Entry = m_Data[i]; if (Entry.DesiredListenPort) { if (Entry.Pid != 0 && IsProcessRunning(Entry.Pid) == false) { ZEN_DEBUG("Sweep - pid {} not running, reclaiming entry (port {})", Entry.Pid.load(), Entry.DesiredListenPort.load()); Entry.Reset(); } } } } void ZenServerState::Snapshot(std::function&& Callback) const { if (m_Data == nullptr) { return; } for (int i = 0; i < m_MaxEntryCount; ++i) { const ZenServerEntry& Entry = m_Data[i]; if (Entry.Pid != 0 && Entry.DesiredListenPort) { if (IsProcessRunning(Entry.Pid.load())) { Callback(Entry); } } } } void ZenServerState::ZenServerEntry::Reset() { Pid = 0; DesiredListenPort = 0; Flags = 0; EffectiveListenPort = 0; } void ZenServerState::ZenServerEntry::SignalShutdownRequest() { Flags |= uint16_t(FlagsEnum::kShutdownPlease); } bool ZenServerState::ZenServerEntry::IsShutdownRequested() const { return (Flags.load() & static_cast(FlagsEnum::kShutdownPlease)) != 0; } void ZenServerState::ZenServerEntry::SignalReady() { Flags |= uint16_t(FlagsEnum::kIsReady); } bool ZenServerState::ZenServerEntry::IsReady() const { return (Flags.load() & static_cast(FlagsEnum::kIsReady)) != 0; } bool ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd) { uint32_t ServerPid = Pid.load(); auto WaitForPickup = [&](uint32_t AddedSlotIndex) { Stopwatch Timer; while (SponsorPids[AddedSlotIndex] == PidToAdd) { // Sponsor processes are checked every second, so 2 second wait time should be enough if (Timer.GetElapsedTimeMs() > 2000) { return false; } if (!IsProcessRunning(ServerPid)) { return false; } Sleep(100); } return true; }; for (uint32_t SponsorIndex = 0; SponsorIndex < 8; SponsorIndex++) { if (SponsorPids[SponsorIndex].load(std::memory_order_relaxed) == PidToAdd) { return WaitForPickup(SponsorIndex); } uint32_t Expected = 0; if (SponsorPids[SponsorIndex].compare_exchange_strong(Expected, PidToAdd)) { // Success! return WaitForPickup(SponsorIndex); } } return false; } ////////////////////////////////////////////////////////////////////////// std::atomic ZenServerTestCounter{0}; ZenServerEnvironment::ZenServerEnvironment() { } ZenServerEnvironment::~ZenServerEnvironment() { } void ZenServerEnvironment::Initialize(std::filesystem::path ProgramBaseDir) { m_ProgramBaseDir = ProgramBaseDir; ZEN_DEBUG("Program base dir is '{}'", ProgramBaseDir); m_IsInitialized = true; } void ZenServerEnvironment::InitializeForTest(std::filesystem::path ProgramBaseDir, std::filesystem::path TestBaseDir, std::string_view ServerClass) { using namespace std::literals; m_ProgramBaseDir = ProgramBaseDir; m_TestBaseDir = TestBaseDir; ZEN_INFO("Program base dir is '{}'", ProgramBaseDir); ZEN_INFO("Cleaning test base dir '{}'", TestBaseDir); DeleteDirectories(TestBaseDir.c_str()); m_IsTestInstance = true; m_IsInitialized = true; if (ServerClass.empty()) { #if ZEN_WITH_HTTPSYS m_ServerClass = "httpsys"sv; #else m_ServerClass = "asio"sv; #endif } else { m_ServerClass = ServerClass; } } std::filesystem::path ZenServerEnvironment::CreateNewTestDir() { using namespace std::literals; ExtendableWideStringBuilder<256> TestDir; TestDir << "test"sv << int64_t(++ZenServerTestCounter); std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str(); ZEN_ASSERT(!std::filesystem::exists(TestPath)); ZEN_INFO("Creating new test dir @ '{}'", TestPath); CreateDirectories(TestPath.c_str()); return TestPath; } std::filesystem::path ZenServerEnvironment::GetTestRootDir(std::string_view Path) { std::filesystem::path Root = m_ProgramBaseDir.parent_path().parent_path(); std::filesystem::path Relative{Path}; return Root / Relative; } ////////////////////////////////////////////////////////////////////////// std::atomic ChildIdCounter{0}; ZenServerInstance::ZenServerInstance(ZenServerEnvironment& TestEnvironment) : m_Env(TestEnvironment) { ZEN_ASSERT(TestEnvironment.IsInitialized()); } ZenServerInstance::~ZenServerInstance() { try { Shutdown(); } catch (const std::exception& Err) { ZEN_ERROR("Shutting down zenserver instance failed, reason: '{}'", Err.what()); } } void ZenServerInstance::SignalShutdown() { m_ShutdownEvent.Set(); } void ZenServerInstance::Shutdown() { if (m_Process.IsValid()) { if (m_ShutdownOnDestroy) { if (m_Terminate) { ZEN_INFO("Terminating zenserver process {}", m_Name); m_Process.Terminate(111); m_Process.Reset(); ZEN_DEBUG("zenserver process {} ({}) terminated", m_Name, m_Process.Pid()); } else { ZEN_DEBUG("Requesting zenserver process {} ({}) to shut down", m_Name, m_Process.Pid()); SignalShutdown(); ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid()); while (!m_Process.Wait(5000)) { ZEN_WARN("Waiting for zenserver process {} ({}) timed out", m_Name, m_Process.Pid()); } m_Process.Reset(); } ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid()); } else { ZEN_DEBUG("Detached from zenserver process {} ({})", m_Name, m_Process.Pid()); } } } void ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs) { ZEN_ASSERT(!m_Process.IsValid()); // Only spawn once const int MyPid = zen::GetCurrentProcessId(); const int ChildId = ++ChildIdCounter; ExtendableStringBuilder<32> ChildEventName; ChildEventName << "Zen_Child_" << ChildId; NamedEvent ChildEvent{ChildEventName}; ExtendableStringBuilder<32> LogId; LogId << "Zen" << ChildId; m_Name = LogId.ToString(); ExtendableStringBuilder<512> CommandLine; CommandLine << "zenserver" ZEN_EXE_SUFFIX_LITERAL; // see CreateProc() call for actual binary path const bool IsTest = m_Env.IsTestEnvironment(); if (IsTest) { if (!m_OwnerPid.has_value()) { m_OwnerPid = MyPid; } CommandLine << " --test --log-id " << m_Name; CommandLine << " --no-sentry"; } if (m_OwnerPid.has_value()) { CommandLine << " --owner-pid " << m_OwnerPid.value(); } CommandLine << " --child-id " << ChildEventName; if (std::string_view ServerClass = m_Env.GetServerClass(); ServerClass.empty() == false) { CommandLine << " --http " << ServerClass; } if (BasePort) { CommandLine << " --port " << BasePort; m_BasePort = gsl::narrow_cast(BasePort); } if (!m_TestDir.empty()) { CommandLine << " --data-dir "; PathToUtf8(m_TestDir.c_str(), CommandLine); } if (!AdditionalServerArgs.empty()) { CommandLine << " " << AdditionalServerArgs; } std::filesystem::path CurrentDirectory = std::filesystem::current_path(); ZEN_DEBUG("Spawning server '{}'", LogId); uint32_t CreationFlags = 0; if (!IsTest) { CreationFlags |= CreateProcOptions::Flag_NewConsole; } const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; CreateProcOptions CreateOptions = { .WorkingDirectory = &CurrentDirectory, .Flags = CreationFlags, }; CreateProcResult ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions); #if ZEN_PLATFORM_WINDOWS if (!ChildPid && ::GetLastError() == ERROR_ELEVATION_REQUIRED) { ZEN_DEBUG("Regular spawn failed - spawning elevated server"); CreateOptions.Flags |= CreateProcOptions::Flag_Elevated; ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions); } #endif if (!ChildPid) { ThrowLastError("Server spawn failed"); } ZEN_DEBUG("Server '{}' spawned OK", LogId); m_Process.Initialize(ChildPid); if (IsTest == false) { DisableShutdownOnDestroy(); } m_ReadyEvent = std::move(ChildEvent); if (WaitTimeoutMs) { if (!WaitUntilReady(WaitTimeoutMs)) { throw std::runtime_error(fmt::format("server start of {} timeout after {}", m_Name, NiceTimeSpanMs(WaitTimeoutMs))); } } } void ZenServerInstance::CreateShutdownEvent(int BasePort) { ExtendableStringBuilder<32> ChildShutdownEventName; ChildShutdownEventName << "Zen_" << BasePort; ChildShutdownEventName << "_Shutdown"; NamedEvent ChildShutdownEvent{ChildShutdownEventName}; m_ShutdownEvent = std::move(ChildShutdownEvent); } void ZenServerInstance::AttachToRunningServer(int BasePort) { ZenServerState State; if (!State.InitializeReadOnly()) { // TODO: return success/error code instead? throw std::runtime_error("No zen state found"); } const ZenServerState::ZenServerEntry* Entry = nullptr; if (BasePort) { Entry = State.Lookup(BasePort); } else { State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) { ZEN_INFO("Found entry pid {}, baseport {}", InEntry.Pid.load(), InEntry.DesiredListenPort.load()); Entry = &InEntry; }); } if (!Entry) { // TODO: return success/error code instead? throw std::runtime_error("No server found"); } ZEN_INFO("Found entry pid {}, baseport {}", Entry->Pid.load(), Entry->DesiredListenPort.load()); std::error_code Ec; m_Process.Initialize(Entry->Pid, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("failed to attach to running server on port {} using pid {}", BasePort, Entry->Pid.load())); } CreateShutdownEvent(Entry->EffectiveListenPort); m_BasePort = Entry->EffectiveListenPort; } void ZenServerInstance::Detach() { if (m_Process.IsValid()) { m_Process.Reset(); m_ShutdownEvent.Close(); } } uint16_t ZenServerInstance::WaitUntilReady() { while (m_ReadyEvent.Wait(100) == false) { if (!m_Process.IsRunning() || !m_Process.IsValid()) { ZEN_WARN("Wait abandoned by invalid process (running={})", m_Process.IsRunning()); return 0; } } OnServerReady(); return m_BasePort; } bool ZenServerInstance::WaitUntilReady(int Timeout) { if (m_ReadyEvent.Wait(Timeout)) { OnServerReady(); return true; } return false; } void ZenServerInstance::OnServerReady() { // Determine effective base port ZenServerState State; if (!State.InitializeReadOnly()) { // TODO: return success/error code instead? throw std::runtime_error("no zen state found"); } const ZenServerState::ZenServerEntry* Entry = nullptr; if (m_BasePort) { Entry = State.Lookup(m_BasePort); } else { State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) { if (InEntry.Pid == (uint32_t)m_Process.Pid()) { Entry = &InEntry; } }); } if (!Entry) { // TODO: return success/error code instead? throw std::runtime_error("no server entry found"); } m_BasePort = Entry->EffectiveListenPort; CreateShutdownEvent(m_BasePort); } std::string ZenServerInstance::GetBaseUri() const { ZEN_ASSERT(m_BasePort); return fmt::format("http://localhost:{}", m_BasePort); } void ZenServerInstance::SetTestDir(std::filesystem::path TestDir) { ZEN_ASSERT(!m_Process.IsValid()); m_TestDir = TestDir; } bool ZenServerInstance::IsRunning() { if (!m_Process.IsValid()) { return false; } return m_Process.IsRunning(); } bool ZenServerInstance::Terminate() { const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; ProcessHandle RunningProcess; std::error_code Ec = FindProcess(Executable, RunningProcess); if (Ec) { throw std::system_error(Ec, fmt::format("failed to look up running server executable '{}'", Executable)); } if (RunningProcess.IsValid()) { if (RunningProcess.Terminate(0)) { return true; } return false; } return true; } CbObject MakeLockFilePayload(const LockFileInfo& Info) { CbObjectWriter Cbo; Cbo << "pid" << Info.Pid << "data" << PathToUtf8(Info.DataDir) << "port" << Info.EffectiveListenPort << "session_id" << Info.SessionId << "ready" << Info.Ready << "executable" << PathToUtf8(Info.ExecutablePath); return Cbo.Save(); } LockFileInfo ReadLockFilePayload(const CbObject& Payload) { LockFileInfo Info; Info.Pid = Payload["pid"].AsInt32(); Info.SessionId = Payload["session_id"].AsObjectId(); Info.EffectiveListenPort = Payload["port"].AsUInt16(); Info.Ready = Payload["ready"].AsBool(); Info.DataDir = Payload["data"].AsU8String(); Info.ExecutablePath = Payload["executable"].AsU8String(); return Info; } bool ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason) { if (Info.Pid == 0) { OutReason = fmt::format("process ({}) is invalid", Info.Pid); return false; } if (!IsProcessRunning(Info.Pid)) { OutReason = fmt::format("process ({}) is not running", Info.Pid); return false; } if (Info.SessionId == Oid::Zero) { OutReason = fmt::format("session id ({}) is not valid", Info.SessionId); return false; } if (Info.EffectiveListenPort == 0) { OutReason = fmt::format("listen port ({}) is not valid", Info.EffectiveListenPort); return false; } if (!std::filesystem::is_directory(Info.DataDir)) { OutReason = fmt::format("data directory ('{}') does not exist", Info.DataDir); return false; } if (!Info.ExecutablePath.empty()) { std::error_code Ec; std::filesystem::path PidPath = GetProcessExecutablePath(Info.Pid, Ec); if (Ec) { OutReason = fmt::format("failed to find executable path of process ('{}'), {}", Info.Pid, Ec.message()); return false; } if (PidPath != Info.ExecutablePath) { OutReason = fmt::format("executable path of process ({}: '{}') does not match executable path '{}'", Info.Pid, PidPath, Info.ExecutablePath); return false; } } return true; } } // namespace zen