path: root/src/zenutil/zenserverprocess.cpp

// Copyright Epic Games, Inc. All Rights Reserved.

#include "zenutil/zenserverprocess.h"

#include <zencore/basicfile.h>
#include <zencore/compactbinary.h>
#include <zencore/compactbinarybuilder.h>
#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/session.h>
#include <zencore/string.h>
#include <zencore/thread.h>
#include <zencore/timer.h>

#include <atomic>

#include <gsl/gsl-lite.hpp>

#if ZEN_PLATFORM_WINDOWS
#	include <zencore/windows.h>
#else
#	include <fcntl.h>
#	include <sys/mman.h>
#	include <sys/stat.h>
#	include <unistd.h>
#endif

//////////////////////////////////////////////////////////////////////////

namespace zen {

// this needs to key off the current process child-id, in order to avoid conflicts
// in situations where we have a tree of zenserver child processes (such as in hub
// tests)

std::atomic<int> ChildIdCounter{0};

void
ZenServerEnvironment::SetBaseChildId(int InitialValue)
{
	ZEN_ASSERT(ChildIdCounter == 0);
	ChildIdCounter = InitialValue;
}

namespace zenutil {
#if ZEN_PLATFORM_WINDOWS
	class SecurityAttributes
	{
	public:
		inline SECURITY_ATTRIBUTES* Attributes() { return &m_Attributes; }

	protected:
		SECURITY_ATTRIBUTES m_Attributes{};
		SECURITY_DESCRIPTOR m_Sd{};
	};

	// Security attributes which allows any user access

	class AnyUserSecurityAttributes : public SecurityAttributes
	{
	public:
		AnyUserSecurityAttributes()
		{
			m_Attributes.nLength		= sizeof m_Attributes;
			m_Attributes.bInheritHandle = false;  // Disable inheritance

			const BOOL Success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION);

			if (Success)
			{
				if (!SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE))
				{
					ThrowLastError("SetSecurityDescriptorDacl failed");
				}

				m_Attributes.lpSecurityDescriptor = &m_Sd;
			}
		}
	};
#endif	// ZEN_PLATFORM_WINDOWS

}  // namespace zenutil

//////////////////////////////////////////////////////////////////////////

ZenServerState::ZenServerState()
{
}

ZenServerState::~ZenServerState()
{
	if (m_OurEntry)
	{
		// Clean up our entry now that we're leaving

		m_OurEntry->Reset();
		m_OurEntry = nullptr;
	}

#if ZEN_PLATFORM_WINDOWS
	if (m_Data)
	{
		UnmapViewOfFile(m_Data);
	}

	if (m_hMapFile)
	{
		CloseHandle(m_hMapFile);
	}
#else
	if (m_Data != nullptr)
	{
		munmap(m_Data, m_MaxEntryCount * sizeof(ZenServerEntry));
	}

	int Fd = int(intptr_t(m_hMapFile));
	close(Fd);
#endif

	m_Data = nullptr;
}

void
ZenServerState::Initialize()
{
	size_t MapSize = m_MaxEntryCount * sizeof(ZenServerEntry);

#if ZEN_PLATFORM_WINDOWS
	// TODO: there's a small chance of a race here, this logic could be tightened up with a mutex to
	//		 ensure only a single process at a time creates the mapping
	// TODO: the fallback to Local instead of Global has a flaw where if you start a non-elevated instance
	//		 first then start an elevated instance second you'll have the first instance with a local
	//		 mapping and the second instance with a global mapping.  This kind of elevated/non-elevated
	//		 shouldn't be common, but handling for it should be improved in the future.

	HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap");
	if (hMap == NULL)
	{
		hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Local\\ZenMap");
	}

	if (hMap == NULL)
	{
		// Security attributes to enable any user to access state
		zenutil::AnyUserSecurityAttributes Attrs;

		hMap = CreateFileMapping(INVALID_HANDLE_VALUE,	// use paging file
								 Attrs.Attributes(),	// allow anyone to access
								 PAGE_READWRITE,		// read/write access
								 0,						// maximum object size (high-order DWORD)
								 DWORD(MapSize),		// maximum object size (low-order DWORD)
								 L"Global\\ZenMap");	// name of mapping object

		if (hMap == NULL)
		{
			hMap = CreateFileMapping(INVALID_HANDLE_VALUE,						// use paging file
									 Attrs.Attributes(),						// allow anyone to access
									 PAGE_READWRITE,							// read/write access
									 0,											// maximum object size (high-order DWORD)
									 m_MaxEntryCount * sizeof(ZenServerEntry),	// maximum object size (low-order DWORD)
									 L"Local\\ZenMap");							// name of mapping object
		}

		if (hMap == NULL)
		{
			ThrowLastError("Could not open or create file mapping object for Zen server state");
		}
	}

	void* pBuf = MapViewOfFile(hMap,				 // handle to map object
							   FILE_MAP_ALL_ACCESS,	 // read/write permission
							   0,					 // offset high
							   0,					 // offset low
							   DWORD(MapSize));

	if (pBuf == NULL)
	{
		ThrowLastError("Could not map view of Zen server state");
	}
#else
	int Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT | O_CLOEXEC, geteuid() == 0 ? 0766 : 0666);
	if (Fd < 0)
	{
		// Work around a potential issue if the service user is changed in certain configurations.
		// If the sysctl 'fs.protected_regular' is set to 1 or 2 (default on many distros),
		// we will be unable to open an existing shared memory object created by another user using O_CREAT,
		// even if we have the correct permissions, or are running as root. If we destroy the existing
		// shared memory object and retry, we'll be able to get past shm_open() so long as we have
		// the appropriate permissions to create the shared memory object.
		shm_unlink("/UnrealEngineZen");
		Fd = shm_open("/UnrealEngineZen", O_RDWR | O_CREAT | O_CLOEXEC, geteuid() == 0 ? 0766 : 0666);
		if (Fd < 0)
		{
			ThrowLastError("Could not open a shared memory object");
		}
	}
	fchmod(Fd, 0666);
	void* hMap = (void*)intptr_t(Fd);

	int Result = ftruncate(Fd, MapSize);
	ZEN_UNUSED(Result);

	void* pBuf = mmap(nullptr, MapSize, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0);
	if (pBuf == MAP_FAILED)
	{
		close(Fd);
		ThrowLastError("Could not map view of Zen server state");
	}
#endif

	m_hMapFile	 = hMap;
	m_Data		 = reinterpret_cast<ZenServerEntry*>(pBuf);
	m_IsReadOnly = false;
}

bool
ZenServerState::InitializeReadOnly()
{
	size_t MapSize = m_MaxEntryCount * sizeof(ZenServerEntry);

#if ZEN_PLATFORM_WINDOWS
	HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap");
	if (hMap == NULL)
	{
		hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Local\\ZenMap");
	}

	if (hMap == NULL)
	{
		return false;
	}

	void* pBuf = MapViewOfFile(hMap,		   // handle to map object
							   FILE_MAP_READ,  // read permission
							   0,			   // offset high
							   0,			   // offset low
							   MapSize);

	if (pBuf == NULL)
	{
		ThrowLastError("Could not map view of Zen server state");
	}
#else
	int Fd = shm_open("/UnrealEngineZen", O_RDONLY | O_CLOEXEC, 0666);
	if (Fd < 0)
	{
		return false;
	}
	void* hMap = (void*)intptr_t(Fd);

	void* pBuf = mmap(nullptr, MapSize, PROT_READ, MAP_SHARED, Fd, 0);
	if (pBuf == MAP_FAILED)
	{
		ThrowLastError("Could not map read-only view of Zen server state");
	}
#endif

	m_hMapFile = hMap;
	m_Data	   = reinterpret_cast<ZenServerEntry*>(pBuf);

	return true;
}

ZenServerState::ZenServerEntry*
ZenServerState::Lookup(int DesiredListenPort) const
{
	for (int i = 0; i < m_MaxEntryCount; ++i)
	{
		uint16_t EntryPort = m_Data[i].DesiredListenPort;
		if (EntryPort != 0)
		{
			if (DesiredListenPort == 0 || (EntryPort == DesiredListenPort))
			{
				std::error_code _;
				if (IsProcessRunning(m_Data[i].Pid, _))
				{
					return &m_Data[i];
				}
			}
		}
	}

	return nullptr;
}

ZenServerState::ZenServerEntry*
ZenServerState::LookupByEffectivePort(int Port) const
{
	for (int i = 0; i < m_MaxEntryCount; ++i)
	{
		uint16_t EntryPort = m_Data[i].EffectiveListenPort;
		if (EntryPort != 0)
		{
			if (EntryPort == Port)
			{
				std::error_code _;
				if (IsProcessRunning(m_Data[i].Pid, _))
				{
					return &m_Data[i];
				}
			}
		}
	}

	return nullptr;
}

ZenServerState::ZenServerEntry*
ZenServerState::Register(int DesiredListenPort)
{
	if (m_Data == nullptr)
	{
		return nullptr;
	}

	// Allocate an entry

	int Pid = GetCurrentProcessId();

	for (int i = 0; i < m_MaxEntryCount; ++i)
	{
		ZenServerEntry& Entry = m_Data[i];

		if (Entry.DesiredListenPort.load(std::memory_order_relaxed) == 0)
		{
			uint16_t Expected = 0;
			if (Entry.DesiredListenPort.compare_exchange_strong(Expected, uint16_t(DesiredListenPort)))
			{
				// Successfully allocated entry

				m_OurEntry = &Entry;

				Entry.Pid				  = Pid;
				Entry.EffectiveListenPort = 0;
				Entry.Flags				  = 0;

				const Oid SesId = GetSessionId();
				memcpy(Entry.SessionId, &SesId, sizeof SesId);

				return &Entry;
			}
		}
	}

	return nullptr;
}

void
ZenServerState::Sweep()
{
	if (m_Data == nullptr)
	{
		return;
	}

	ZEN_ASSERT(m_IsReadOnly == false);

	for (int i = 0; i < m_MaxEntryCount; ++i)
	{
		ZenServerEntry& Entry = m_Data[i];

		if (Entry.DesiredListenPort)
		{
			std::error_code ErrorCode;
			if (Entry.Pid != 0 && IsProcessRunning(Entry.Pid, ErrorCode) == false)
			{
				if (ErrorCode)
				{
					ZEN_WARN("Sweep - can not determine running state for pid {}, skipping entry (port {}). Reason: '{}'",
							 Entry.Pid.load(),
							 Entry.DesiredListenPort.load(),
							 ErrorCode.message());
				}
				else
				{
					ZEN_DEBUG("Sweep - pid {} not running, reclaiming entry (port {})", Entry.Pid.load(), Entry.DesiredListenPort.load());
					Entry.Reset();
				}
			}
		}
	}
}

void
ZenServerState::Snapshot(std::function<void(const ZenServerEntry&)>&& Callback) const
{
	if (m_Data == nullptr)
	{
		return;
	}

	for (int i = 0; i < m_MaxEntryCount; ++i)
	{
		const ZenServerEntry& Entry = m_Data[i];

		if (Entry.Pid != 0 && Entry.DesiredListenPort)
		{
			std::error_code ErrorCode;
			if (IsProcessRunning(Entry.Pid.load(), ErrorCode))
			{
				if (ErrorCode)
				{
					ZEN_WARN("Snapshot - can not determine running state for pid {}, skipping entry (port {}). Reason: '{}'",
							 Entry.Pid.load(),
							 Entry.DesiredListenPort.load(),
							 ErrorCode.message());
				}
				else
				{
					Callback(Entry);
				}
			}
		}
	}
}

void
ZenServerState::ZenServerEntry::Reset()
{
	Pid					= 0;
	DesiredListenPort	= 0;
	Flags				= 0;
	EffectiveListenPort = 0;
}

void
ZenServerState::ZenServerEntry::SignalShutdownRequest()
{
	Flags |= uint16_t(FlagsEnum::kShutdownPlease);
}

bool
ZenServerState::ZenServerEntry::IsShutdownRequested() const
{
	return (Flags.load() & static_cast<uint16_t>(FlagsEnum::kShutdownPlease)) != 0;
}

void
ZenServerState::ZenServerEntry::SignalReady()
{
	Flags |= uint16_t(FlagsEnum::kIsReady);
}

bool
ZenServerState::ZenServerEntry::IsReady() const
{
	return (Flags.load() & static_cast<uint16_t>(FlagsEnum::kIsReady)) != 0;
}

bool
ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd, uint64_t Timeout)
{
	uint32_t ServerPid	   = Pid.load();
	auto	 WaitForPickup = [&](uint32_t AddedSlotIndex) {
		if (Timeout == 0)
		{
			return true;
		}
		Stopwatch Timer;
		while (SponsorPids[AddedSlotIndex] == PidToAdd)
		{
			if (Timer.GetElapsedTimeMs() > Timeout)
			{
				SponsorPids[AddedSlotIndex].compare_exchange_strong(PidToAdd, 0);
				return false;
			}
			std::error_code _;
			if (!IsProcessRunning(ServerPid, _))
			{
				SponsorPids[AddedSlotIndex].compare_exchange_strong(PidToAdd, 0);
				return false;
			}
			Sleep(100);
		}
		return true;
	};
	for (uint32_t SponsorIndex = 0; SponsorIndex < 8; SponsorIndex++)
	{
		if (SponsorPids[SponsorIndex].load(std::memory_order_relaxed) == PidToAdd)
		{
			return WaitForPickup(SponsorIndex);
		}
		uint32_t Expected = 0;
		if (SponsorPids[SponsorIndex].compare_exchange_strong(Expected, PidToAdd))
		{
			return WaitForPickup(SponsorIndex);
		}
	}
	return false;
}

//////////////////////////////////////////////////////////////////////////

std::atomic<int> ZenServerTestCounter{0};

ZenServerEnvironment::ZenServerEnvironment()
{
}

ZenServerEnvironment::~ZenServerEnvironment()
{
}

void
ZenServerEnvironment::Initialize(std::filesystem::path ProgramBaseDir)
{
	m_ProgramBaseDir = ProgramBaseDir;

	ZEN_DEBUG("Program base dir is '{}'", ProgramBaseDir);

	m_IsInitialized = true;
}

void
ZenServerEnvironment::InitializeForTest(std::filesystem::path ProgramBaseDir,
										std::filesystem::path TestBaseDir,
										std::string_view	  ServerClass)
{
	using namespace std::literals;

	m_ProgramBaseDir	  = ProgramBaseDir;
	m_ChildProcessBaseDir = TestBaseDir;

	ZEN_INFO("Program base dir is '{}'", ProgramBaseDir);
	ZEN_INFO("Cleaning test base dir '{}'", TestBaseDir);
	DeleteDirectories(TestBaseDir.c_str());

	m_IsTestInstance = true;
	m_IsInitialized	 = true;

	if (ServerClass.empty())
	{
#if ZEN_WITH_HTTPSYS
		if (!zen::windows::IsRunningOnWine())
		{
			m_ServerClass = "httpsys"sv;

			return;
		}
#endif

		m_ServerClass = "asio"sv;
	}
	else
	{
		m_ServerClass = ServerClass;
	}
}

void
ZenServerEnvironment::InitializeForHub(std::filesystem::path ProgramBaseDir,
									   std::filesystem::path ChildBaseDir,
									   std::string_view		 ServerClass)
{
	using namespace std::literals;

	m_ProgramBaseDir	  = ProgramBaseDir;
	m_ChildProcessBaseDir = ChildBaseDir;

	ZEN_INFO("Program base dir is '{}'", m_ProgramBaseDir);
	ZEN_INFO("Cleaning child base dir '{}'", m_ChildProcessBaseDir);
	DeleteDirectories(m_ChildProcessBaseDir.c_str());

	m_IsHubInstance = true;
	m_IsInitialized = true;

	if (ServerClass.empty())
	{
#if ZEN_WITH_HTTPSYS
		if (!zen::windows::IsRunningOnWine())
		{
			m_ServerClass = "httpsys"sv;

			return;
		}
#endif

		m_ServerClass = "asio"sv;
	}
	else
	{
		m_ServerClass = ServerClass;
	}
}

std::filesystem::path
ZenServerEnvironment::CreateChildDir(std::string_view ChildName)
{
	using namespace std::literals;

	std::filesystem::path ChildPath = m_ChildProcessBaseDir / ChildName;

	if (!IsDir(ChildPath))
	{
		ZEN_INFO("Creating new test dir @ '{}'", ChildPath);

		CreateDirectories(ChildPath);
	}

	return ChildPath;
}

std::filesystem::path
ZenServerEnvironment::CreateNewTestDir()
{
	using namespace std::literals;

	ExtendableWideStringBuilder<256> TestDir;
	TestDir << "test"sv << int64_t(ZenServerTestCounter.fetch_add(1));

	std::filesystem::path TestPath = m_ChildProcessBaseDir / TestDir.c_str();
	ZEN_ASSERT(!IsDir(TestPath));

	ZEN_INFO("Creating new test dir @ '{}'", TestPath);

	CreateDirectories(TestPath.c_str());

	return TestPath;
}

std::filesystem::path
ZenServerEnvironment::GetTestRootDir(std::string_view Path)
{
	std::filesystem::path Root = m_ProgramBaseDir.parent_path().parent_path();

	std::filesystem::path Relative{Path};

	return Root / Relative;
}

//////////////////////////////////////////////////////////////////////////

ZenServerInstance::ZenServerInstance(ZenServerEnvironment& TestEnvironment, ServerMode Mode) : m_Env(TestEnvironment), m_ServerMode(Mode)
{
	ZEN_ASSERT(TestEnvironment.IsInitialized());

	m_ServerMode = Mode;
}

ZenServerInstance::~ZenServerInstance()
{
	try
	{
		Shutdown();
		std::error_code DummyEc;
		RemoveFile(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc);
	}
	catch (const std::exception& Err)
	{
		ZEN_ERROR("Shutting down zenserver instance failed, reason: '{}'", Err.what());
	}
}

bool
ZenServerInstance::SignalShutdown(std::error_code& OutEc)
{
	if (m_ShutdownEvent)
	{
		OutEc = m_ShutdownEvent->Set();
		return !OutEc;
	}
	else
	{
		return false;
	}
}

int
ZenServerInstance::Shutdown()
{
	if (m_Process.IsValid())
	{
		if (m_ShutdownOnDestroy)
		{
			if (m_Terminate)
			{
				ZEN_INFO("Terminating zenserver process {}", m_Name);
				int ExitCode = 111;
				m_Process.Terminate(ExitCode);
				ZEN_DEBUG("zenserver process {} ({}) terminated", m_Name, m_Process.Pid());
				return ExitCode;
			}
			else
			{
				if (!m_Process.IsRunning())
				{
					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
					int ExitCode = m_Process.GetExitCode();
					m_Process.Reset();
					return ExitCode;
				}

				ZEN_DEBUG("Requesting zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
				std::error_code Ec;
				if (SignalShutdown(Ec))
				{
					Stopwatch Timer;
					ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
					while (!m_Process.Wait(2000))
					{
						if (!m_Process.IsValid())
						{
							ZEN_WARN("Wait abandoned by invalid process");
							break;
						}
						if (!m_Process.IsRunning())
						{
							ZEN_WARN("Wait abandoned by exited process");
							return 0;
						}
						ZEN_WARN("Waited for zenserver process {} ({}) to exit for {}",
								 m_Name,
								 m_Process.Pid(),
								 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
					}
					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
					int ExitCode = m_Process.GetExitCode();
					m_Process.Reset();
					return ExitCode;
				}
				else if (Ec)
				{
					ZEN_WARN("Terminating zenserver process as we failed to signal zenserver process {} ({}) to shut down. Reason: '{}'",
							 m_Name,
							 m_Process.Pid(),
							 Ec.message());
				}
				else
				{
					ZEN_INFO("Terminating zenserver process as we did not wait for it to get ready {}", m_Name);
				}

				int ExitCode = 111;
				m_Process.Terminate(ExitCode);
				ZEN_DEBUG("zenserver process {} ({}) terminated", m_Name, m_Process.Pid());
				return ExitCode;
			}
		}
		else
		{
			if (m_Process.Wait(0))
			{
				int ExitCode = m_Process.GetExitCode();
				ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
				m_Process.Reset();
				return ExitCode;
			}
			ZEN_DEBUG("Detached from zenserver process {} ({})", m_Name, m_Process.Pid());
			return 0;
		}
	}
	return -1;
}

int
ZenServerInstance::AssignName()
{
	const int ChildId = ++ChildIdCounter;

	ExtendableStringBuilder<32> LogId;
	LogId << "Zen" << ChildId;
	m_Name = LogId.ToString();

	return ChildId;
}

void
ZenServerInstance::SpawnServer(std::string_view ServerArgs, bool OpenConsole, int WaitTimeoutMs)
{
	ZEN_ASSERT(!m_Process.IsValid());  // Only spawn once

	const int ChildId = AssignName();

	SpawnServerInternal(ChildId, ServerArgs, OpenConsole, WaitTimeoutMs);
}

std::string_view
ToString(ZenServerInstance::ServerMode Mode)
{
	using namespace std::literals;

	switch (Mode)
	{
		case ZenServerInstance::ServerMode::kStorageServer:
			return "storage"sv;
		case ZenServerInstance::ServerMode::kHubServer:
			return "hub"sv;
		default:
			return "invalid"sv;
	}
}

void
ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, bool OpenConsole, int WaitTimeoutMs)
{
	const bool IsTest = m_Env.IsTestEnvironment();

	ExtendableStringBuilder<32> ChildEventName;
	ChildEventName << "Zen_Child_" << ChildId;
	NamedEvent ChildEvent{ChildEventName};

	ExtendableStringBuilder<512> CommandLine;
	CommandLine << "zenserver" ZEN_EXE_SUFFIX_LITERAL;	// see CreateProc() call for actual binary path

	if (m_ServerMode == ServerMode::kHubServer)
	{
		CommandLine << " hub";
	}

	CommandLine << " --child-id " << ChildEventName;

	if (!ServerArgs.empty())
	{
		CommandLine << " " << ServerArgs;
	}

	std::filesystem::path CurrentDirectory = std::filesystem::current_path();

	ZEN_DEBUG("Spawning {} server '{}'", ToString(m_ServerMode), m_Name);

	uint32_t CreationFlags = 0;
	if (OpenConsole)
	{
		CreationFlags |= CreateProcOptions::Flag_NewConsole;
	}

	const std::filesystem::path BaseDir = m_Env.ProgramBaseDir();
	const std::filesystem::path Executable =
		m_ServerExecutablePath.empty() ? (BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL) : m_ServerExecutablePath;
	const std::filesystem::path OutputPath =
		OpenConsole ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log");
	CreateProcOptions CreateOptions = {
		.WorkingDirectory = &CurrentDirectory,
		.Flags			  = CreationFlags,
		.StdoutFile		  = OutputPath,
#if ZEN_PLATFORM_WINDOWS
		.AssignToJob = m_JobObject,
#endif
	};
	CreateProcResult ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions);
#if ZEN_PLATFORM_WINDOWS
	if (!ChildPid)
	{
		DWORD Error = GetLastError();
		if (Error == ERROR_ELEVATION_REQUIRED)
		{
			ZEN_DEBUG("Regular spawn failed - spawning elevated server");
			CreateOptions.Flags |= CreateProcOptions::Flag_Elevated;
			// ShellExecuteEx (used by the elevated path) does not support job object assignment
			if (CreateOptions.AssignToJob)
			{
				ZEN_WARN("Elevated process spawn does not support job object assignment; child will not be auto-terminated on parent exit");
				CreateOptions.AssignToJob = nullptr;
			}
			ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions);
		}
		else
		{
			ThrowSystemError(Error, "Server spawn failed");
		}
	}
#endif

	if (!ChildPid)
	{
		ThrowLastError("Server spawn failed");
	}

	ZEN_DEBUG("Server '{}' spawned OK (pid:{})", m_Name, GetProcessId(ChildPid));

	m_Process.Initialize(ChildPid);

	if (IsTest == false)
	{
		DisableShutdownOnDestroy();
	}

	m_ReadyEvent = std::move(ChildEvent);

	if (WaitTimeoutMs)
	{
		if (!WaitUntilReady(WaitTimeoutMs))
		{
			throw std::runtime_error(fmt::format("server start of {} {} after {}: {}",
												 m_Name,
												 m_Process.IsRunning() ? "timeout" : "crash",
												 NiceTimeSpanMs(WaitTimeoutMs),
												 GetLogOutput()));
		}
	}
}

void
ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs)
{
	ZEN_ASSERT(!m_Process.IsValid());  // Only spawn once

	const int MyPid	  = zen::GetCurrentProcessId();
	const int ChildId = AssignName();

	ExtendableStringBuilder<512> CommandLine;
	const bool					 IsTest = m_Env.IsTestEnvironment();

	if (IsTest)
	{
		if (!m_OwnerPid.has_value())
		{
			m_OwnerPid = MyPid;
		}

		CommandLine << " --test --log-id " << m_Name;
		CommandLine << " --no-sentry";

		if (AdditionalServerArgs.find("--system-dir") == std::string_view::npos)
		{
			CommandLine << " --system-dir ";
			PathToUtf8((m_Env.CreateNewTestDir() / "system-dir").c_str(), CommandLine);
		}
	}

	if (m_OwnerPid.has_value())
	{
		CommandLine << " --owner-pid " << m_OwnerPid.value();
	}

	if (std::string_view ServerClass = m_Env.GetServerClass(); ServerClass.empty() == false)
	{
		CommandLine << " --http " << ServerClass;
	}

	if (BasePort)
	{
		CommandLine << " --port " << BasePort;
		m_BasePort = gsl::narrow_cast<uint16_t>(BasePort);
	}

	if (!m_DataDir.empty())
	{
		CommandLine << " --data-dir ";
		PathToUtf8(m_DataDir.c_str(), CommandLine);
	}

	if (!AdditionalServerArgs.empty())
	{
		CommandLine << " " << AdditionalServerArgs;
	}

	const bool OpenConsole = !IsTest && !m_Env.IsHubEnvironment();
	SpawnServerInternal(ChildId, CommandLine, OpenConsole, WaitTimeoutMs);
}

void
ZenServerInstance::CreateShutdownEvent(int BasePort)
{
	ExtendableStringBuilder<32> ChildShutdownEventName;
	ChildShutdownEventName << "Zen_" << BasePort;
	ChildShutdownEventName << "_Shutdown";
	m_ShutdownEvent = std::make_unique<NamedEvent>(ChildShutdownEventName);
}

void
ZenServerInstance::AttachToRunningServer(int BasePort)
{
	ZenServerState State;
	if (!State.InitializeReadOnly())
	{
		// TODO: return success/error code instead?
		throw std::runtime_error("No zen state found");
	}

	const ZenServerState::ZenServerEntry* Entry = nullptr;

	if (BasePort)
	{
		Entry = State.Lookup(BasePort);
	}
	else
	{
		State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) {
			ZEN_INFO("Found entry pid {}, baseport {}", InEntry.Pid.load(), InEntry.DesiredListenPort.load());
			Entry = &InEntry;
		});
	}

	if (!Entry)
	{
		// TODO: return success/error code instead?
		throw std::runtime_error("No server found");
	}

	ZEN_INFO("Found entry pid {}, baseport {}", Entry->Pid.load(), Entry->DesiredListenPort.load());

	std::error_code Ec;
	m_Process.Initialize(Entry->Pid, Ec);

	if (Ec)
	{
		throw std::system_error(Ec, fmt::format("failed to attach to running server on port {} using pid {}", BasePort, Entry->Pid.load()));
	}

	CreateShutdownEvent(Entry->EffectiveListenPort);
	m_BasePort = Entry->EffectiveListenPort;
}

void
ZenServerInstance::Detach()
{
	if (m_Process.IsValid())
	{
		m_Process.Reset();
		m_ShutdownEvent.reset();
	}
}

uint16_t
ZenServerInstance::WaitUntilReady()
{
	while (m_ReadyEvent.Wait(10) == false)
	{
		if (!m_Process.IsValid())
		{
			ZEN_WARN("Wait abandoned by invalid process");
			return 0;
		}
		if (!m_Process.IsRunning())
		{
			ZEN_WARN("Wait abandoned by exited process");
			return 0;
		}
	}

	OnServerReady();

	return m_BasePort;
}

bool
ZenServerInstance::WaitUntilReady(int Timeout)
{
	int TimeoutLeftMS = Timeout;
	while (m_ReadyEvent.Wait(10) == false)
	{
		if (!m_Process.IsValid())
		{
			ZEN_WARN("Wait abandoned by invalid process");
			return false;
		}
		if (!m_Process.IsRunning())
		{
			ZEN_WARN("Wait abandoned by exited process");
			return false;
		}
		TimeoutLeftMS -= 10;
		if ((TimeoutLeftMS <= 0))
		{
			ZEN_WARN("Wait abandoned due to timeout");
			return false;
		}
	}

	OnServerReady();

	return true;
}

bool
ZenServerInstance::WaitUntilExited(int Timeout, std::error_code& OutEc)
{
	if (m_Process.IsRunning())
	{
		return m_Process.Wait(Timeout, OutEc);
	}
	return false;
}

void
ZenServerInstance::OnServerReady()
{
	// Determine effective base port

	ZenServerState State;
	if (!State.InitializeReadOnly())
	{
		// TODO: return success/error code instead?
		throw std::runtime_error("no zen state found");
	}

	const ZenServerState::ZenServerEntry* Entry = nullptr;

	if (m_BasePort)
	{
		Entry = State.Lookup(m_BasePort);
	}
	else
	{
		State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) {
			if (InEntry.Pid == (uint32_t)m_Process.Pid())
			{
				Entry = &InEntry;
			}
		});
	}

	if (!Entry)
	{
		// TODO: return success/error code instead?
		throw std::runtime_error("no server entry found");
	}
	ZEN_ASSERT(Entry->IsReady());

	m_BasePort = Entry->EffectiveListenPort;
	ZEN_ASSERT(m_BasePort != 0);
	if (!IsProcessRunning(Entry->Pid.load()))
	{
		throw std::runtime_error("server no longer running");
	}
	CreateShutdownEvent(m_BasePort);

	ZEN_DEBUG("Server '{}' is ready on port {}", m_Name, m_BasePort);
}

std::string
ZenServerInstance::GetBaseUri() const
{
	ZEN_ASSERT(m_BasePort);

	return fmt::format("http://localhost:{}", m_BasePort);
}

void
ZenServerInstance::SetDataDir(std::filesystem::path TestDir)
{
	ZEN_ASSERT(!m_Process.IsValid());
	m_DataDir = TestDir;
}

bool
ZenServerInstance::IsRunning()
{
	if (!m_Process.IsValid())
	{
		return false;
	}
	return m_Process.IsRunning();
}

std::string
ZenServerInstance::GetLogOutput() const
{
	std::filesystem::path OutputPath = std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log");
	if (IsFile(OutputPath))
	{
		FileContents Contents = ReadFile(OutputPath);
		if (!Contents.ErrorCode)
		{
			IoBuffer Content = Contents.Flatten();
			if (Content)
			{
				std::string Log((const char*)Content.Data(), Content.Size());
				return Log;
			}
		}
	}
	return {};
}

bool
ZenServerInstance::Terminate()
{
	const std::filesystem::path BaseDir	   = m_Env.ProgramBaseDir();
	const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
	ProcessHandle				RunningProcess;
	std::error_code				Ec = FindProcess(Executable, RunningProcess, /*IncludeSelf*/ false);
	if (Ec)
	{
		throw std::system_error(Ec, fmt::format("failed to look up running server executable '{}'", Executable));
	}
	if (RunningProcess.IsValid())
	{
		if (RunningProcess.Terminate(0))
		{
			return true;
		}
		return false;
	}
	return true;
}

CbObject
MakeLockFilePayload(const LockFileInfo& Info)
{
	CbObjectWriter Cbo;
	Cbo << "pid" << Info.Pid << "data" << PathToUtf8(Info.DataDir) << "port" << Info.EffectiveListenPort << "session_id" << Info.SessionId
		<< "ready" << Info.Ready << "executable" << PathToUtf8(Info.ExecutablePath);
	return Cbo.Save();
}
LockFileInfo
ReadLockFilePayload(const CbObject& Payload)
{
	LockFileInfo Info;
	Info.Pid				 = Payload["pid"].AsInt32();
	Info.SessionId			 = Payload["session_id"].AsObjectId();
	Info.EffectiveListenPort = Payload["port"].AsUInt16();
	Info.Ready				 = Payload["ready"].AsBool();
	Info.DataDir			 = Payload["data"].AsU8String();
	Info.ExecutablePath		 = Payload["executable"].AsU8String();
	return Info;
}

bool
ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason)
{
	if (Info.Pid == 0)
	{
		OutReason = fmt::format("process ({}) is invalid", Info.Pid);
		return false;
	}
	std::error_code ErrorCode;
	if (!IsProcessRunning(Info.Pid, ErrorCode))
	{
		if (ErrorCode)
		{
			OutReason = fmt::format("process ({}) can not be checked. Reason: '{}'", Info.Pid, ErrorCode.message());
		}
		else
		{
			OutReason = fmt::format("process ({}) is not running", Info.Pid);
		}
		return false;
	}
	if (Info.SessionId == Oid::Zero)
	{
		OutReason = fmt::format("session id ({}) is not valid", Info.SessionId);
		return false;
	}
	if (Info.EffectiveListenPort == 0)
	{
		OutReason = fmt::format("listen port ({}) is not valid", Info.EffectiveListenPort);
		return false;
	}
	if (!IsDir(Info.DataDir))
	{
		OutReason = fmt::format("data directory ('{}') does not exist", Info.DataDir);
		return false;
	}
	if (!Info.ExecutablePath.empty())
	{
		std::error_code		  Ec;
		std::filesystem::path PidPath = GetProcessExecutablePath(Info.Pid, Ec);
		if (Ec)
		{
			OutReason = fmt::format("failed to find executable path of process ('{}'), {}", Info.Pid, Ec.message());
			return false;
		}
		if (PidPath != Info.ExecutablePath)
		{
			OutReason = fmt::format("executable path of process ({}: '{}') does not match executable path '{}'",
									Info.Pid,
									PidPath,
									Info.ExecutablePath);
			return false;
		}
	}
	return true;
}

}  // namespace zen