From 1ed3139e577f6c8aa6d07f7e76afa3a80d9d4852 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 27 Feb 2026 19:36:22 +0100 Subject: Add test summary table and failure reporting to xmake test (#794) - Add a summary table printed after all test suites complete, showing per-suite test case counts, assertion counts, timings and pass/fail status. - Add failure reporting: individual failing test cases are listed at the end with their file path and line number for easy navigation. - Made zenserver instances spawned by a hub not create new console windows for a better background testing experience - The TestListener in testing.cpp now writes a machine-readable summary file (via `ZEN_TEST_SUMMARY_FILE` env var) containing aggregate counts and per-test-case failure details. This runs as a doctest listener alongside any active reporter, so it works with both console and JUnit modes. - Tests now run in a deterministic order defined by a single ordered list that also serves as the test name/target mapping, replacing the previous unordered table + separate order list. - The `--run` option now accepts comma-separated values (e.g. `--run=core,http,util`) and validates each name, reporting unknown test names early. - Fix platform detection in `xmake test`: the config command now passes `-p` explicitly, fixing "mingw" misdetection when running from Git Bash on Windows. - Add missing "util" entry to the help text for `--run`. --- src/zenutil/zenserverprocess.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/zenutil/zenserverprocess.cpp') diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index ef2a4fda5..579ba450a 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -934,7 +934,8 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr CommandLine << " " << AdditionalServerArgs; } - SpawnServerInternal(ChildId, CommandLine, !IsTest, WaitTimeoutMs); + const bool OpenConsole = !IsTest && !m_Env.IsHubEnvironment(); + SpawnServerInternal(ChildId, CommandLine, OpenConsole, WaitTimeoutMs); } void -- cgit v1.2.3 From f796ee9e650d5f73844f862ed51a6de6bb33c219 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 28 Feb 2026 15:36:50 +0100 Subject: subprocess tracking using Jobs on Windows/hub (#796) This change introduces job object support on Windows to be able to more accurately track and limit resource usage on storage instances created by the hub service. It also ensures that all child instances can be torn down reliably on exit. Also made it so hub tests no longer pop up console windows while running. --- src/zenutil/zenserverprocess.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'src/zenutil/zenserverprocess.cpp') diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index 579ba450a..0f8ab223d 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -831,8 +831,15 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, m_ServerExecutablePath.empty() ? (BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL) : m_ServerExecutablePath; const std::filesystem::path OutputPath = OpenConsole ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); - CreateProcOptions CreateOptions = {.WorkingDirectory = &CurrentDirectory, .Flags = CreationFlags, .StdoutFile = OutputPath}; - CreateProcResult ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions); + CreateProcOptions CreateOptions = { + .WorkingDirectory = &CurrentDirectory, + .Flags = CreationFlags, + .StdoutFile = OutputPath, +#if ZEN_PLATFORM_WINDOWS + .AssignToJob = m_JobObject, +#endif + }; + CreateProcResult ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions); #if ZEN_PLATFORM_WINDOWS if (!ChildPid) { @@ -841,6 +848,12 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, { ZEN_DEBUG("Regular spawn failed - spawning elevated server"); CreateOptions.Flags |= CreateProcOptions::Flag_Elevated; + // ShellExecuteEx (used by the elevated path) does not support job object assignment + if (CreateOptions.AssignToJob) + { + ZEN_WARN("Elevated process spawn does not support job object assignment; child will not be auto-terminated on parent exit"); + CreateOptions.AssignToJob = nullptr; + } ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions); } else -- cgit v1.2.3 From 4d01aaee0a45f4c9f96e8a4925eff696be98de8d Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 1 Mar 2026 12:40:20 +0100 Subject: added `--verbose` option to zenserver-test and `xmake test` (#798) * when `--verbose` is specified to zenserver-test, all child process output (typically, zenserver instances) is piped through to stdout. you can also pass `--verbose` to `xmake test` to accomplish the same thing. * this PR also consolidates all test runner `main` function logic (such as from zencore-test, zenhttp-test etc) into central implementation in zencore for consistency and ease of maintenance * also added extended utf8-tests including a fix to `Utf8ToWide()` --- src/zenutil/zenserverprocess.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'src/zenutil/zenserverprocess.cpp') diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index 0f8ab223d..e127a92d7 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -829,12 +829,13 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); const std::filesystem::path Executable = m_ServerExecutablePath.empty() ? (BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL) : m_ServerExecutablePath; - const std::filesystem::path OutputPath = - OpenConsole ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); - CreateProcOptions CreateOptions = { - .WorkingDirectory = &CurrentDirectory, - .Flags = CreationFlags, - .StdoutFile = OutputPath, + const std::filesystem::path OutputPath = (OpenConsole || m_Env.IsPassthroughOutput()) + ? std::filesystem::path{} + : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); + CreateProcOptions CreateOptions = { + .WorkingDirectory = &CurrentDirectory, + .Flags = CreationFlags, + .StdoutFile = OutputPath, #if ZEN_PLATFORM_WINDOWS .AssignToJob = m_JobObject, #endif -- cgit v1.2.3 From 0763d09a81e5a1d3df11763a7ec75e7860c9510a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 14:13:46 +0100 Subject: compute orchestration (#763) - Added local process runners for Linux/Wine, Mac with some sandboxing support - Horde & Nomad provisioning for development and testing - Client session queues with lifecycle management (active/draining/cancelled), automatic retry with configurable limits, and manual reschedule API - Improved web UI for orchestrator, compute, and hub dashboards with WebSocket push updates - Some security hardening - Improved scalability and `zen exec` command Still experimental - compute support is disabled by default --- src/zenutil/zenserverprocess.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/zenutil/zenserverprocess.cpp') diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index e127a92d7..b09c2d89a 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -787,6 +787,8 @@ ToString(ZenServerInstance::ServerMode Mode) return "storage"sv; case ZenServerInstance::ServerMode::kHubServer: return "hub"sv; + case ZenServerInstance::ServerMode::kComputeServer: + return "compute"sv; default: return "invalid"sv; } @@ -808,6 +810,10 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, { CommandLine << " hub"; } + else if (m_ServerMode == ServerMode::kComputeServer) + { + CommandLine << " compute"; + } CommandLine << " --child-id " << ChildEventName; -- cgit v1.2.3 From e9d04250225a430cffed28e6a49299e3da542f97 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 11 Mar 2026 09:45:31 +0100 Subject: hub consul integration (#820) - Feature: Basic consul integration for zenserver hub mode, restricted to host local consul agent and register/deregister of services - Feature: Added new options to zenserver hub mode - `consul-endpoint` - Consul endpoint URL for service registration (empty = disabled) - `hub-base-port-number` - Base port number for provisioned instances - `hub-instance-limit` - Maximum number of provisioned instances for this hub - `hub-use-job-object` - Enable the use of a Windows Job Object for child process management (Windows only) --- src/zenutil/zenserverprocess.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'src/zenutil/zenserverprocess.cpp') diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index b09c2d89a..b9c50be4f 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -503,6 +503,39 @@ ZenServerEnvironment::~ZenServerEnvironment() { } +ZenServerEnvironment::ZenServerEnvironment(ZenServerEnvironment&& Other) +: m_ProgramBaseDir(std::move(Other.m_ProgramBaseDir)) +, m_ChildProcessBaseDir(std::move(Other.m_ChildProcessBaseDir)) +, m_IsInitialized(Other.m_IsInitialized) +, m_IsTestInstance(Other.m_IsTestInstance) +, m_IsHubInstance(Other.m_IsHubInstance) +, m_PassthroughOutput(Other.m_PassthroughOutput) +, m_ServerClass(std::move(Other.m_ServerClass)) +, m_NextPortNumber(Other.m_NextPortNumber.load()) +{ +} + +ZenServerEnvironment::ZenServerEnvironment(EStorageTag, std::filesystem::path ProgramBaseDir) +{ + Initialize(ProgramBaseDir); +} + +ZenServerEnvironment::ZenServerEnvironment(EHubTag, + std::filesystem::path ProgramBaseDir, + std::filesystem::path TestBaseDir, + std::string_view ServerClass) +{ + InitializeForHub(ProgramBaseDir, TestBaseDir, ServerClass); +} + +ZenServerEnvironment::ZenServerEnvironment(ETestTag, + std::filesystem::path ProgramBaseDir, + std::filesystem::path TestBaseDir, + std::string_view ServerClass) +{ + InitializeForTest(ProgramBaseDir, TestBaseDir, ServerClass); +} + void ZenServerEnvironment::Initialize(std::filesystem::path ProgramBaseDir) { -- cgit v1.2.3 From ef586f5930ac761f8e8e18cde2ebd5248efeaa4a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 13 Mar 2026 09:24:59 +0100 Subject: Unix Domain Socket auto discovery (#833) This PR adds end-to-end Unix domain socket (UDS) support, allowing zen CLI to discover and connect to UDS-only servers automatically. - **`unix://` URI scheme in zen CLI**: The `-u` / `--hosturl` option now accepts `unix:///path/to/socket` to connect to a zenserver via a Unix domain socket instead of TCP. - **Per-instance shared memory for extended server info**: Each zenserver instance now publishes a small shared memory section (keyed by SessionId) containing per-instance data that doesn't fit in the fixed-size ZenServerEntry -- starting with the UDS socket path. This is a 4KB pagefile-backed section on Windows (`Global\ZenInstance_{sessionid}`) and a POSIX shared memory object on Linux/Mac (`/UnrealEngineZen_{sessionid}`). - **Client-side auto-discovery of UDS servers**: `zen info`, `zen status`, etc. now automatically discover and prefer UDS connections when a server publishes a socket path. Servers running with `--no-network` (UDS-only) are no longer invisible to the CLI. - **`kNoNetwork` flag in ZenServerEntry**: Servers started with `--no-network` advertise this in their shared state entry. Clients skip TCP fallback for these servers, and display commands (`ps`, `status`, `top`) show `-` instead of a port number to indicate TCP is not available. --- src/zenutil/zenserverprocess.cpp | 247 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 246 insertions(+), 1 deletion(-) (limited to 'src/zenutil/zenserverprocess.cpp') diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index b9c50be4f..ac614f779 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -449,6 +449,30 @@ ZenServerState::ZenServerEntry::IsReady() const return (Flags.load() & static_cast(FlagsEnum::kIsReady)) != 0; } +void +ZenServerState::ZenServerEntry::SignalHasInstanceInfo() +{ + Flags |= uint16_t(FlagsEnum::kHasInstanceInfo); +} + +bool +ZenServerState::ZenServerEntry::HasInstanceInfo() const +{ + return (Flags.load() & static_cast(FlagsEnum::kHasInstanceInfo)) != 0; +} + +void +ZenServerState::ZenServerEntry::SignalNoNetwork() +{ + Flags |= uint16_t(FlagsEnum::kNoNetwork); +} + +bool +ZenServerState::ZenServerEntry::IsNoNetwork() const +{ + return (Flags.load() & static_cast(FlagsEnum::kNoNetwork)) != 0; +} + bool ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd, uint64_t Timeout) { @@ -491,6 +515,222 @@ ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd, uint64_t Ti return false; } +////////////////////////////////////////////////////////////////////////// +// ZenServerInstanceInfo +////////////////////////////////////////////////////////////////////////// + +static constexpr size_t kInstanceInfoSize = 4096; + +ZenServerInstanceInfo::ZenServerInstanceInfo() = default; + +ZenServerInstanceInfo::~ZenServerInstanceInfo() +{ +#if ZEN_PLATFORM_WINDOWS + if (m_Data) + { + UnmapViewOfFile(m_Data); + } + if (m_hMapFile) + { + CloseHandle(m_hMapFile); + } +#else + if (m_Data != nullptr) + { + munmap(m_Data, kInstanceInfoSize); + } + if (m_hMapFile != nullptr) + { + int Fd = int(intptr_t(m_hMapFile)); + close(Fd); + } + if (m_IsOwner) + { + std::string Name = MakeName(m_SessionId); + shm_unlink(Name.c_str()); + } +#endif + m_Data = nullptr; +} + +std::string +ZenServerInstanceInfo::MakeName(const Oid& SessionId) +{ +#if ZEN_PLATFORM_WINDOWS + return fmt::format("Global\\ZenInstance_{}", SessionId); +#else + // macOS limits shm_open names to ~31 chars (PSHMNAMLEN), so keep this short. + // "/ZenI_" (6) + 24 hex = 30 chars, within the limit. + return fmt::format("/ZenI_{}", SessionId); +#endif +} + +void +ZenServerInstanceInfo::Create(const Oid& SessionId, const InstanceInfoData& Data) +{ + m_SessionId = SessionId; + m_IsOwner = true; + + // Serialize the data to compact binary + CbObjectWriter Cbo; + if (!Data.UnixSocketPath.empty()) + { + Cbo << "unix_socket" << PathToUtf8(Data.UnixSocketPath); + } + CbObject Payload = Cbo.Save(); + + MemoryView PayloadView = Payload.GetView(); + uint32_t PayloadSize = gsl::narrow(PayloadView.GetSize()); + + std::string Name = MakeName(SessionId); + +#if ZEN_PLATFORM_WINDOWS + zenutil::AnyUserSecurityAttributes Attrs; + + std::wstring WideName(Name.begin(), Name.end()); + + HANDLE hMap = + CreateFileMappingW(INVALID_HANDLE_VALUE, Attrs.Attributes(), PAGE_READWRITE, 0, DWORD(kInstanceInfoSize), WideName.c_str()); + + if (hMap == NULL) + { + // Fall back to Local namespace + std::string LocalName = fmt::format("Local\\ZenInstance_{}", SessionId); + std::wstring WideLocalName(LocalName.begin(), LocalName.end()); + hMap = CreateFileMappingW(INVALID_HANDLE_VALUE, + Attrs.Attributes(), + PAGE_READWRITE, + 0, + DWORD(kInstanceInfoSize), + WideLocalName.c_str()); + } + + if (hMap == NULL) + { + ThrowLastError("Could not create instance info shared memory"); + } + + void* pBuf = MapViewOfFile(hMap, FILE_MAP_ALL_ACCESS, 0, 0, DWORD(kInstanceInfoSize)); + if (pBuf == NULL) + { + CloseHandle(hMap); + ThrowLastError("Could not map instance info shared memory"); + } +#else + int Fd = shm_open(Name.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, 0666); + if (Fd < 0) + { + ThrowLastError("Could not create instance info shared memory"); + } + fchmod(Fd, 0666); + + if (ftruncate(Fd, kInstanceInfoSize) < 0) + { + close(Fd); + shm_unlink(Name.c_str()); + ThrowLastError("Could not resize instance info shared memory"); + } + + void* pBuf = mmap(nullptr, kInstanceInfoSize, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0); + if (pBuf == MAP_FAILED) + { + close(Fd); + shm_unlink(Name.c_str()); + ThrowLastError("Could not map instance info shared memory"); + } + + void* hMap = reinterpret_cast(intptr_t(Fd)); +#endif + + m_hMapFile = hMap; + m_Data = reinterpret_cast(pBuf); + + // Write payload: [uint32_t size][compact binary bytes] + memcpy(m_Data, &PayloadSize, sizeof PayloadSize); + if (PayloadSize > 0) + { + memcpy(m_Data + sizeof(uint32_t), PayloadView.GetData(), PayloadSize); + } +} + +bool +ZenServerInstanceInfo::OpenReadOnly(const Oid& SessionId) +{ + m_SessionId = SessionId; + + std::string Name = MakeName(SessionId); + +#if ZEN_PLATFORM_WINDOWS + std::wstring WideName(Name.begin(), Name.end()); + + HANDLE hMap = OpenFileMappingW(FILE_MAP_READ, FALSE, WideName.c_str()); + if (hMap == NULL) + { + // Fall back to Local namespace + std::string LocalName = fmt::format("Local\\ZenInstance_{}", SessionId); + std::wstring WideLocalName(LocalName.begin(), LocalName.end()); + hMap = OpenFileMappingW(FILE_MAP_READ, FALSE, WideLocalName.c_str()); + } + + if (hMap == NULL) + { + return false; + } + + void* pBuf = MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, DWORD(kInstanceInfoSize)); + if (pBuf == NULL) + { + CloseHandle(hMap); + return false; + } +#else + int Fd = shm_open(Name.c_str(), O_RDONLY | O_CLOEXEC, 0666); + if (Fd < 0) + { + return false; + } + + void* pBuf = mmap(nullptr, kInstanceInfoSize, PROT_READ, MAP_SHARED, Fd, 0); + if (pBuf == MAP_FAILED) + { + close(Fd); + return false; + } + + void* hMap = reinterpret_cast(intptr_t(Fd)); +#endif + + m_hMapFile = hMap; + m_Data = reinterpret_cast(pBuf); + m_IsOwner = false; + + return true; +} + +InstanceInfoData +ZenServerInstanceInfo::Read() const +{ + InstanceInfoData Result; + + if (m_Data == nullptr) + { + return Result; + } + + uint32_t PayloadSize = 0; + memcpy(&PayloadSize, m_Data, sizeof PayloadSize); + + if (PayloadSize == 0 || PayloadSize > kInstanceInfoSize - sizeof(uint32_t)) + { + return Result; + } + + CbObject Payload = CbObject::Clone(m_Data + sizeof(uint32_t)); + Result.UnixSocketPath = Payload["unix_socket"].AsU8String(); + + return Result; +} + ////////////////////////////////////////////////////////////////////////// std::atomic ZenServerTestCounter{0}; @@ -1234,6 +1474,10 @@ MakeLockFilePayload(const LockFileInfo& Info) CbObjectWriter Cbo; Cbo << "pid" << Info.Pid << "data" << PathToUtf8(Info.DataDir) << "port" << Info.EffectiveListenPort << "session_id" << Info.SessionId << "ready" << Info.Ready << "executable" << PathToUtf8(Info.ExecutablePath); + if (!Info.UnixSocketPath.empty()) + { + Cbo << "unix_socket" << PathToUtf8(Info.UnixSocketPath); + } return Cbo.Save(); } LockFileInfo @@ -1246,6 +1490,7 @@ ReadLockFilePayload(const CbObject& Payload) Info.Ready = Payload["ready"].AsBool(); Info.DataDir = Payload["data"].AsU8String(); Info.ExecutablePath = Payload["executable"].AsU8String(); + Info.UnixSocketPath = Payload["unix_socket"].AsU8String(); return Info; } @@ -1275,7 +1520,7 @@ ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason) OutReason = fmt::format("session id ({}) is not valid", Info.SessionId); return false; } - if (Info.EffectiveListenPort == 0) + if (Info.EffectiveListenPort == 0 && Info.UnixSocketPath.empty()) { OutReason = fmt::format("listen port ({}) is not valid", Info.EffectiveListenPort); return false; -- cgit v1.2.3