From 4598f710da2d0e387c53eb97d983ff520c211a8f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 9 Apr 2026 10:34:30 +0200 Subject: migrate from http_parser to llhttp (#929) --- xmake.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake.lua') diff --git a/xmake.lua b/xmake.lua index 30f3d3411..b3232b820 100644 --- a/xmake.lua +++ b/xmake.lua @@ -208,7 +208,7 @@ set_policy("package.precompiled", false) add_defines("gsl_FEATURE_GSL_COMPATIBILITY_MODE=1") add_requires("gsl-lite", {system = false}) -add_requires("http_parser", {system = false}) +add_requires("llhttp", {system = false}) add_requires("json11", {system = false}) add_requires("lua", {system = false}) add_requires("lz4", {system = false}) -- cgit v1.2.3 From e009b73a69aae5b3492e9bc9c03f13cb9b388c86 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 11 Apr 2026 15:17:53 +0200 Subject: removed s3 test program (#942) Remove the `zens3-testbed` target and source files. This was a standalone test harness for S3 operations that is no longer needed. --- xmake.lua | 1 - 1 file changed, 1 deletion(-) (limited to 'xmake.lua') diff --git a/xmake.lua b/xmake.lua index b3232b820..d2e061852 100644 --- a/xmake.lua +++ b/xmake.lua @@ -501,7 +501,6 @@ end if has_config("zennomad") then includes("src/zennomad") end -includes("src/zens3-testbed") includes("src/zenstore", "src/zenstore-test") includes("src/zentelemetry", "src/zentelemetry-test") includes("src/zenutil", "src/zenutil-test") -- cgit v1.2.3 From eb677209c0b837c3797c951ec1eb937965e40418 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 20 Apr 2026 14:10:10 +0200 Subject: zen-test: add CLI integration harness + TestArtifactProvider + CI host stats (#985) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Establishes a new end-to-end integration test harness for the `zen` CLI, the shared fetcher it uses to pull test artifacts, and the CI plumbing that feeds both. Also lowers the default test-harness log level and broadens the artifact fetcher's credential resolution. ### `zen-test` executable (`src/zen-test/`) - New binary modeled on `zenserver-test`, built only in debug. - `zen-test.{h,cpp}` harness: spawns `zen.exe` via `CreateProc` and captures combined stdout/stderr into a `ZenCommandResult` for assertion. - Registered with `scripts/test.lua` under the short name `zen` (`xmake test --run=zen`) and enabled for `--kill-stale-processes`. - Prints a clear console message when invoked from a release build (tests disabled), so misconfiguration is easy to spot. - Documented in `CLAUDE.md` (test-suite naming table + test projects section) and `README.md`. - Test cases in the `zen.artifactprovider` suite: - `probe.lyra_cook_rpc_recording` — probe against a canonical Lyra cook RPC recording that skips with a diagnostic `MESSAGE` when no artifact source is configured. - `probe.s3_readme` — probes the configured S3 bucket for `README.md` using a fresh temp cache to force the request through to S3; skips on macOS without static creds (no EC2 Mac runners in our fleet). - `zen.utility-cmd` suite: new integration tests exercising `zen print`, `zen wipe`, and `zen copy`. ### `TestArtifactProvider` (`src/zenutil/testartifactprovider.{h,cpp}`) - `Ref` factory returning a local-only or S3-backed provider, selected from env vars: - `ZEN_TEST_ARTIFACTS_PATH` — local directory to serve from (write-through cache for remote fetches). - `ZEN_TEST_ARTIFACTS_S3` — S3 URL to fetch from. - `AWS_DEFAULT_REGION` / `AWS_REGION`, `AWS_ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` — standard AWS config. - `Exists(path)` / `Fetch(path)` API with a `TestArtifactFetchResult` return carrying the content buffer and a diagnostic error string. Content is cached on disk across test runs. - **IMDS credential fallback**: when no static `AWS_ACCESS_KEY_ID` is present, attaches an `ImdsCredentialProvider` so self-hosted EC2 runners with an attached IAM role can sign S3 requests without static credentials (mirrors the pattern in `zenserver/hub/hydration.cpp`). - **IMDS opt-out**: honors the standard `AWS_EC2_METADATA_DISABLED=true` env var, and skips IMDS by default on macOS where the link-local probe would just emit noise. ### Test harness log level (`src/zencore/testing.cpp`) - `TestRunner::ApplyCommandLine` now defaults the global log level to `Info` (was effectively `Trace`), cutting the noise from `xmake test --run=all` now that the suite has grown. Applies uniformly to `zencore-test`, `zenhttp-test`, `zenstore-test`, `zenutil-test`, `zenserver-test`, `zen-test`, etc. `--debug` (Debug) and `--verbose` (Trace) still opt back in when chasing failures. ### CI (`.github/workflows/validate.yml`) - **Runner info step** on all three platforms (Windows/Linux/macOS): prints host, CPU topology, memory, and disk usage before the build/test step, so flakes that correlate with a particular runner or low disk space are easy to spot. - **Artifact env wiring**: passes `ZEN_TEST_ARTIFACTS_S3` and `AWS_DEFAULT_REGION` into the debug Build & Test step on all three platforms so the probe can reach its source when the repo variable is configured. The probe skips cleanly when unset. --- xmake.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake.lua') diff --git a/xmake.lua b/xmake.lua index d2e061852..13f6fdaca 100644 --- a/xmake.lua +++ b/xmake.lua @@ -508,7 +508,7 @@ if is_plat("windows") then includes("src/zenvfs") end includes("src/zenserver", "src/zenserver-test") -includes("src/zen") +includes("src/zen", "src/zen-test") includes("src/zentest-appstub") -------------------------------------------------------------------------- -- cgit v1.2.3 From 2dfb5da16b97a6c12e01977af5b5188522178a4e Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 20 Apr 2026 21:50:41 +0200 Subject: zen trace analysis support (#945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrates the **tourist** trace analysis library and builds a full `zen trace` command suite for working with Unreal Engine `.utrace` files. ### Trace analysis library (`thirdparty/tourist/`) - Adds the tourist library as a third-party dependency with three modules: **foundation** (platform primitives, memory, scheduling), **trace** (UE Trace protocol decoding), and **analysis** (event dispatching and analyzer framework). - Cross-platform support for Windows, Linux, and macOS. ### `zen trace` CLI commands (`src/zen/cmds/`, `src/zen/trace/`) - **`zen trace analyze`** — Summarize a `.utrace` file: session metadata, thread inventory, command line + build configuration, CPU profiling scopes, timing, event rates, log messages, and (with symbols) memory allocation metrics including live-allocs dumps, callstack-keyed aggregation, and allocation churn. Optional HTML output for memory reports. - **`zen trace inspect`** — Dump the event schema (declared types, fields, sizes) from a trace file. - **`zen trace trim`** — Extract a time-window from a trace into a new `.utrace` file. - **`zen trace serve`** — Launch a local HTTP server hosting an interactive trace viewer; opens in the default browser. ### Symbolication (`src/zen/trace/symbol_resolver.*`, `thirdparty/raw_pdb/`) - Pluggable resolver with multiple backends: `pdb` (in-tree raw_pdb), `dbghelp` (Windows), `llvm-symbolizer` (all platforms), `atos` (macOS). An `auto` backend picks the best available tool per platform. - Microsoft Symbol Server support: downloads PDBs on demand using a redirect-aware HTTP client. - Local PDB cache keyed by image GUID preserves symbols across binary recompilation. - Callstack trimming heuristic strips UE internal noise from reports. - Binary analysis cache (`.ucache_z`) avoids re-resolving the same trace. ### Interactive trace viewer (`src/zen/frontend/html/`, `src/zen/trace/trace_viewer_service.*`) - Timeline: scope-level detail, horizontal zoom/pan, vertical scrolling, viewport-driven loading with pre-computed LOD for responsive navigation of large traces. - Thread grouping (collapsible sidebar sections) synthesized from name suffixes, natural sort order, visual distinction between lane threads and OS threads. - Bookmark and region annotations; region categories with per-category toggles; bookmark marker toggle in the toolbar. - Filterable Logs tab showing captured `UE_LOG` output. - Stats tab with per-scope aggregate statistics. - Memory tab with interactive allocation analysis and an allocation size histogram. - CsvProfiler event parsing and chart UI. ### Other in-branch supporting changes - **Cross-platform browser launcher** (`browser_launcher.{h,cpp}`) used by `trace serve`. - **`ReciprocalU64`** fast 64-bit integer division (zencore/intmath) for trace analyzers. - **`parallelsort`** cross-platform parallel sort helper (zenutil). - Frontend zip build rule so the viewer's HTML assets are bundled into `zen.exe`. - `/Zo` flag for better optimized debug info on Windows release builds. - `trace-tests.cpp` in the `zen-test` harness (harness itself landed on main via #985). --- xmake.lua | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'xmake.lua') diff --git a/xmake.lua b/xmake.lua index 13f6fdaca..27abf9b09 100644 --- a/xmake.lua +++ b/xmake.lua @@ -329,6 +329,10 @@ if is_os("windows") then add_cxxflags("/Zc:u8EscapeEncoding") -- Enable UTF-8 encoding for u8 string literals (clang does this by default) add_cxxflags("/Zc:preprocessor") -- Enable preprocessor conformance mode add_cxxflags("/Zc:inline") -- Enforce inline semantics + if is_mode("release") then + add_cflags("/Zo") -- better debug info for optimized code + add_cxxflags("/Zo") -- better debug info for optimized code + end end -- add_ldflags("/MAP") @@ -462,6 +466,7 @@ option("zentrace") option_end() add_define_by_config("ZEN_WITH_TRACE", "zentrace") + set_warnings("allextra", "error") set_languages("cxx20") -- cgit v1.2.3 From 245d2e562165d048e5ee2ab97f1260975a8142d3 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 21 Apr 2026 13:48:41 +0200 Subject: zen CLI security review fixes (#974) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security review follow-ups to the `zen` CLI. Each fix stands on its own commit. Grouped by category below. ## Credentials and secrets - **Per-install random auth encryption key instead of a hardcoded literal.** The default AES key and IV used to encrypt persisted OIDC refresh tokens / OAuth client secrets were ASCII literals compiled into the public source. Replaced with 32+16 random bytes persisted to `/auth/machinekey.dat`. `SecureRandomBytes` added in zencore/crypto wrapping BCryptGenRandom / OpenSSL / mbedTLS CTR_DRBG. Partial override (only one of `--encryption-aes-key`/`--encryption-aes-iv`) is now rejected instead of silently using the hardcoded half. - **Wrap the machine key with OS-protected storage.** `machinekey.dat` is now a tagged format (4-byte magic + flags + wrapped-or-raw payload). Windows wraps via DPAPI (`CryptProtectData` at per-user scope) so a stolen disk copy cannot decrypt without the OS master key. macOS uses Keychain Services (GenericPassword under `org.unrealengine.zen.auth`, `kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly`). Linux uses libsecret (opt-in via `--zenlibsecret=yes`, off by default because headless servers typically have no Secret Service daemon). All platforms fall back to raw persistence with `0600` perms on POSIX when wrapping is unavailable. Legacy files from the prior commit are detected by size and still read. > Note: argv-redaction before Sentry on crash was previously part of this PR but was superseded by `ScrubSensitiveValues()` from #989; this PR now just calls that helper instead of walking argv itself. ## Path traversal - **Reject unsafe filenames from the remote oplog in `oplog-mirror`.** The filename from each oplog entry was joined to the mirror root without normalisation; a compromised remote could use drive letters, UNC shares, device path prefixes, absolute paths, or `..` components to write anywhere the zen user could write. An `UnsafeFileNameReason` check runs immediately after extraction, logs the offending filename, and aborts the mirror. - **Use the resolved absolute download-spec path in `builds download`.** `--download-spec-path` was computed into a sanitised absolute path, then the original unresolved path was passed to `ParseBuildManifest`, bypassing the `MakeSafeAbsolutePath` mitigations and reading from the process cwd rather than `--local-path`. ## Input validation - **Stop asserting on malformed `--build-id` / `--build-part-id`.** `Oid::FromHexString` asserts on bad input and `ZEN_ASSERT` is active in release, so a too-short or non-hex user value aborted the process instead of surfacing an `OptionParseException`. Routed all callers through `TryFromHexString`. Also fixes `ParseBuildPartId` reporting errors under the wrong option name. - **Check the JSON parse error in `oplog-export --builds-metadata-path`.** The single-arg `LoadCompactBinaryFromJson` overload discarded the parser error; malformed JSON shipped a truncated compact-binary `metadata` field to the server with no indication. Switched to the two-arg overload and throws a descriptive error naming the file and reason. - **Format the actual value in the malformed `--url` error.** The message was constructed with a literal `{}` placeholder and no `fmt::format` call, so users saw the placeholder instead of the offending URL. - **Require `--output-path` in `cache get` unless `--as-text` is set.** Previously an empty path auto-filled from the value key / attachment hash and wrote into the process cwd; the `--as-text && empty path` stdout branch was unreachable because the auto-fill ran first. - **Clear the cxxopts `allow_unrecognised_options` flag after permissive parse.** `ParseOptionsPermissive` set the flag on the Options it received and never cleared it, priming that Options for silent typo acceptance on any later reuse. Added `disallow_unrecognised_options()` to the vendored cxxopts (local patch — flagged at the declaration) and wrapped the toggle in RAII. ## Resource lifecycle - **Restore signal handlers via RAII.** `wipe`, `builds`, and `oplog-mirror` installed SIGINT/SIGBREAK handlers with raw `signal()` and never restored them; an option-parse throw left the handler targeting an abort flag nothing reads. Added `zen::ScopedSignalHandler` in zen.h and applied at all three sites (builds uses `std::optional` members so the guards survive past `OnParentOptionsParsed` into the subcommand's Run). - **Route SIGINT in `oplog-mirror` to the worker-pool abort flag.** The command declared a local `std::atomic AbortFlag` but no handler targeted it — Ctrl-C killed the process instead of cleanly aborting. Added a `MirrorAbortFlag` / `MirrorSignalCallbackHandler` pair in projectstore_impl and bound the local as a reference; existing `.store`/`.load`/capture sites unchanged. - **Clean up the `cache get` temp download on every exit path.** `Http.Download` parks the payload in the system temp dir; a failed `MoveToFile` (cross-volume, denied target) or an exception could leave the temp file behind. The downloaded buffer is already flagged delete-on-close by `HttpClient`, so the fix is just to clear that flag after a successful `MoveToFile` so the renamed-out file isn't reaped. ## Other - **Fix wrong URL fields in `oplog-export` / `oplog-import` builds-branch descriptions.** Two operator-facing "[builds] URL/namespace/bucket/buildsid" messages formatted `m_CloudUrl` instead of `m_BuildsUrl` / `m_BuildsHost` (copy-paste from neighbouring `[cloud]` branches), shown as empty or stale at the start of an export/import. - **Fix "Can't find oplog in project '{}'" formatting and a "Failed top mirror" typo in projectstore_cmd.** - **Fix a misleading `oplog-export` comment on the `--zen` scheme default** ("Assume https" vs. the `http://` the code writes). - **Fail `ScrambleDir` when `RemoveFile` doesn't delete.** The `zen builds test` scramble phase used `(void)RemoveFile(FilePath)`, discarding both the bool return and the error. A quiet delete failure let verification run against stale state; switched to the two-arg overload and throw on false return or non-empty `error_code`. --- xmake.lua | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'xmake.lua') diff --git a/xmake.lua b/xmake.lua index 27abf9b09..8200f6991 100644 --- a/xmake.lua +++ b/xmake.lua @@ -408,6 +408,13 @@ option("zenrpmalloc") option_end() add_define_by_config("ZEN_USE_RPMALLOC", "zenrpmalloc") +option("zenlibsecret") + set_default(false) + set_showmenu(true) + set_description("Use libsecret (Secret Service / gnome-keyring) on Linux for zen auth key wrapping. Requires libsecret-1-dev at build time.") +option_end() +add_define_by_config("ZEN_USE_LIBSECRET", "zenlibsecret") + if is_os("windows") then option("httpsys") set_default(true) -- cgit v1.2.3