From 794f093057c58c4909a0653edb54fdf869560596 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 14:00:34 +0100 Subject: native xmake toolchain definition for UE-clang (#805) This change is meant to provide a smoother experience when working on Linux. After this change, the toolchain setup process is now simply ```bash $ scripts/ue_build_linux/get_ue_toolchain.sh ``` and then at config time the toolchain is automatically detected if you downloaded it to the default location or have the `UE_TOOLCHAIN_DIR` environment variable set ```bash xmake config --mode=debug ``` Compared to the old script-based approach this configures the toolchain more precisely, avoiding leakage into unrelated build processes such as when a package manager decides to build something like Ninja locally etc. --- scripts/bundle.lua | 10 +++++++ scripts/ue_build_linux/README.md | 42 +++++++++++++++--------------- scripts/ue_build_linux/clang | 2 -- scripts/ue_build_linux/clang++ | 2 -- scripts/ue_build_linux/get_ue_toolchain.sh | 2 +- scripts/ue_build_linux/ue_build.sh | 33 ----------------------- 6 files changed, 32 insertions(+), 59 deletions(-) delete mode 100755 scripts/ue_build_linux/clang delete mode 100755 scripts/ue_build_linux/clang++ delete mode 100755 scripts/ue_build_linux/ue_build.sh (limited to 'scripts') diff --git a/scripts/bundle.lua b/scripts/bundle.lua index 07e120d04..b2a5e1e08 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -17,7 +17,15 @@ end -------------------------------------------------------------------------------- local function _build(arch, debug, config_args) + import("core.project.config") + config.load() + variant = debug and "debug" or "release" + + -- Preserve toolchain/sdk from current config so --clean doesn't lose them + local toolchain_arg = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or nil + local sdk_arg = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or nil + local ret = _exec( "xmake", "config", @@ -26,6 +34,8 @@ local function _build(arch, debug, config_args) "--mode="..variant, "--arch="..arch, "--zensentry=yes", + toolchain_arg, + sdk_arg, config_args) if ret > 0 then raise("Failed to configure xmake") diff --git a/scripts/ue_build_linux/README.md b/scripts/ue_build_linux/README.md index e93a234ae..afafcbe24 100755 --- a/scripts/ue_build_linux/README.md +++ b/scripts/ue_build_linux/README.md @@ -2,38 +2,38 @@ This folder contains scripts to build Zen using the UE Linux toolchain. This can be used to output binaries that meet the VFX Reference Platform versions. -It works by using the `--sysroot=` option to redirect compilers and linkers to -find headers and libraries. There are a few components involved; -1) get_ue_toolchain.sh +## Setup + +Download the toolchain using `get_ue_toolchain.sh`: ``` -$ scripts/ue_build_linux/get_ue_toolchain.sh ./.tmp-ue-toolchain +$ scripts/ue_build_linux/get_ue_toolchain.sh ``` -This will download the required components from cdn.unrealengine.com and -structure them in such a way that they can be used by both vcpkg and xmake -when building Zen. +By default this downloads to `~/.ue-toolchain`. A custom path can be given as +the first argument, or via the `UE_TOOLCHAIN_DIR` environment variable. -2) ue_build.sh [args...] +This will download the required components from cdn.unrealengine.com and +structure them in such a way that they can be used by xmake when building Zen. -Given the toolchain location downloaded in step (1) and the `VCPKG_ROOT` -environment variable is properly configured, this script sets up a suitable -environment and execs the "prog [args...]". +## Building -It is expected that this is used to invoke xmake to build Zen; +xmake automatically detects the toolchain at `~/.ue-toolchain`, so no extra +flags are needed: ``` -$ scripts/ue_build_linux/ue_build.sh .tmp-ue-toolchain xmake config --mode=debug -$ scripts/ue_build_linux/ue_build.sh .tmp-ue-toolchain xmake build +$ xmake config -y -m debug +$ xmake build -y ``` -It is possible that `--toolchain=clang` may be required as a configuration -option. The `ue_build.sh` script can also be sourced into the current shell, -although it is worth noting that this has never been tried. +To build a release bundle: -3) `scripts/ue_build_linux/clang` / `scripts/ue_build_linux/clang++` +``` +$ xmake config -y -m release +$ xmake bundle -y +``` -These acts as shims to the binaries in `toolchain_dir`, adding in the required -command line arguments to use the correct headers and libraries. -The `ue_build.sh` script adjusts `$PATH` appropriately. +The toolchain can also be selected explicitly with `--toolchain=ue-clang`, and +the SDK location can be overridden with `--sdk=` (must be absolute) or +the `UE_TOOLCHAIN_DIR` environment variable. diff --git a/scripts/ue_build_linux/clang b/scripts/ue_build_linux/clang deleted file mode 100755 index 9666ba4ba..000000000 --- a/scripts/ue_build_linux/clang +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec $UE_TOOLCHAIN_DIR/bin/clang --sysroot=$UE_TOOLCHAIN_DIR $CFLAGS "$@" diff --git a/scripts/ue_build_linux/clang++ b/scripts/ue_build_linux/clang++ deleted file mode 100755 index be106ae87..000000000 --- a/scripts/ue_build_linux/clang++ +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec $UE_TOOLCHAIN_DIR/bin/clang++ --sysroot=$UE_TOOLCHAIN_DIR -stdlib=libc++ $CXXFLAGS "$@" diff --git a/scripts/ue_build_linux/get_ue_toolchain.sh b/scripts/ue_build_linux/get_ue_toolchain.sh index c2538b09a..0afd40fe3 100755 --- a/scripts/ue_build_linux/get_ue_toolchain.sh +++ b/scripts/ue_build_linux/get_ue_toolchain.sh @@ -5,7 +5,7 @@ ZEN_ROOT=$(realpath $SCRIPT_DIR/../..) die() { echo "ERROR: $1"; exit; } -toolchain_dir="${1:-${ZEN_ROOT}/.tmp-ue-toolchain}" +toolchain_dir="${1:-${UE_TOOLCHAIN_DIR:-${HOME}/.ue-toolchain}}" if [[ $toolchain_dir == "--help" ]]; then echo "usage: $(basename ${BASH_SOURCE[0]}) " diff --git a/scripts/ue_build_linux/ue_build.sh b/scripts/ue_build_linux/ue_build.sh deleted file mode 100755 index 690f9f661..000000000 --- a/scripts/ue_build_linux/ue_build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -ZEN_ROOT=$(realpath $SCRIPT_DIR/../..) - -die() { echo ERROR: $1; exit 1; } - -if [[ $1 == "--help" ]]; then - echo "usage: $0 (defaults to ${ZEN_ROOT}/.tmp-ue-toolchain)" - exit -fi - -toolchain_dir="${1:-${ZEN_ROOT}/.tmp-ue-toolchain}" - -# Validate input - -if ! [ -d $toolchain_dir ]; then - die "$1 is not a directory" -fi - -if ! [ -e $toolchain_dir/bin/clang++ ]; then - die "$1/bin/clang++ does not exist" -fi - -export UE_TOOLCHAIN_DIR=$(realpath $toolchain_dir) -export CC="clang" -export CXX="clang++" -export LD="clang++" - -export PATH="$(realpath $(dirname ${BASH_SOURCE[0]})):$PATH" - -shift -exec "$@" -- cgit v1.2.3 From 2f0d60cb431ffefecf3e0a383528691be74af21b Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 5 Mar 2026 14:31:27 +0100 Subject: oidctoken tool package (#810) * added OidcToken binary to the build process. The binary is mirrored from p4 and is placed next to the output of the build process. It is also placed in the release zip archives. * also fixed issue with Linux symbol stripping which was introduced in toolchain changes yesterday --- scripts/bundle.lua | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/bundle.lua b/scripts/bundle.lua index b2a5e1e08..6f540c5b8 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -199,8 +199,9 @@ local function main_windows(signidentity) "build/windows/x64/release/zenserver.pdb", "build/windows/x64/release/zen.exe", "build/windows/x64/release/zen.pdb", - "build/windows/x64/release/crashpad_handler.exe") -end + "build/windows/x64/release/crashpad_handler.exe", + "build/windows/x64/release/oidctoken.exe") + end -------------------------------------------------------------------------------- local function main_mac(signidentity) @@ -277,7 +278,8 @@ local function main_mac(signidentity) "build/zenserver-macos.zip", "build/macosx/universal/release/zenserver", "build/macosx/universal/release/zen", - "build/macosx/universal/release/crashpad_handler") + "build/macosx/universal/release/crashpad_handler", + "build/macosx/x86_64/release/Oidctoken") end -------------------------------------------------------------------------------- @@ -288,7 +290,8 @@ local function main_linux() "build/zenserver-linux.zip", "build/linux/x86_64/release/zenserver", "build/linux/x86_64/release/zen", - "build/linux/x86_64/release/crashpad_handler") + "build/linux/x86_64/release/crashpad_handler", + "build/linux/x86_64/release/Oidctoken") end -------------------------------------------------------------------------------- -- cgit v1.2.3 From f9d8cbcb3573b47b639b7bd73d3a4eed17653d71 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Mar 2026 13:08:00 +0100 Subject: add fallback for zencache multirange (#816) * clean up BuildStorageResolveResult to allow capabilities * add check for multirange request capability * add MaxRangeCountPerRequest capabilities * project export tests * add InMemoryBuildStorageCache * progress and logging improvements * fix ElapsedSeconds calculations in fileremoteprojectstore.cpp * oplogs/builds test script --- .../test_scripts/builds-download-upload-test.py | 196 ++++++++++++++++++ scripts/test_scripts/metadatas/AndroidClient.json | 9 + scripts/test_scripts/metadatas/IOSClient.json | 9 + scripts/test_scripts/metadatas/LinuxServer.json | 9 + scripts/test_scripts/metadatas/PS4Client.json | 9 + scripts/test_scripts/metadatas/Switch2Client.json | 9 + scripts/test_scripts/metadatas/SwitchClient.json | 9 + scripts/test_scripts/metadatas/WindowsClient.json | 9 + scripts/test_scripts/metadatas/XB1Client.json | 9 + scripts/test_scripts/oplog-import-export-test.py | 228 +++++++++++++++++++++ 10 files changed, 496 insertions(+) create mode 100644 scripts/test_scripts/builds-download-upload-test.py create mode 100644 scripts/test_scripts/metadatas/AndroidClient.json create mode 100644 scripts/test_scripts/metadatas/IOSClient.json create mode 100644 scripts/test_scripts/metadatas/LinuxServer.json create mode 100644 scripts/test_scripts/metadatas/PS4Client.json create mode 100644 scripts/test_scripts/metadatas/Switch2Client.json create mode 100644 scripts/test_scripts/metadatas/SwitchClient.json create mode 100644 scripts/test_scripts/metadatas/WindowsClient.json create mode 100644 scripts/test_scripts/metadatas/XB1Client.json create mode 100644 scripts/test_scripts/oplog-import-export-test.py (limited to 'scripts') diff --git a/scripts/test_scripts/builds-download-upload-test.py b/scripts/test_scripts/builds-download-upload-test.py new file mode 100644 index 000000000..f03528d98 --- /dev/null +++ b/scripts/test_scripts/builds-download-upload-test.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Test script for builds download/upload operations.""" + +from __future__ import annotations + +import argparse +import platform +import subprocess +import sys +from pathlib import Path +from typing import NamedTuple + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" + + +class Build(NamedTuple): + name: str + bucket: str + id: str + + +BUILDS = [ + Build("XB1Client", "fortnitegame.staged-build.fortnite-main.xb1-client", "09a7616c1a388dfe6056aa57"), + Build("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client", "09a762c81e2cf213142d0ce5"), + Build("SwitchClient", "fortnitegame.staged-build.fortnite-main.switch-client", "09a75bf9c3ce75bce09f644f"), + Build("LinuxServer", "fortnitegame.staged-build.fortnite-main.linux-server", "09a750ac155eb3e3b62e87e0"), + Build("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client", "09a78f3df07b289691ec5710"), + Build("PS4Client", "fortnitegame.staged-build.fortnite-main.ps4-client", "09a76ea92ad301d4724fafad"), + Build("IOSClient", "fortnitegame.staged-build.fortnite-main.ios-client", "09a7816fa26c23362fef0c5d"), + Build("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client", "09a76725f1620d62c6be06e4"), +] + +ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") +ZEN_METADATA_DIR: Path = Path(__file__).resolve().parent / "metadatas" + +ZEN_PORT = 8558 +ZEN_CACHE_PORT = 8559 +ZEN_CACHE = f"http://127.0.0.1:{ZEN_CACHE_PORT}" +ZEN_PARTIAL_REQUEST_MODE = "true" + +SERVER_ARGS: tuple[str, ...] = ( + "--http", "asio", + "--gc-cache-duration-seconds", "1209600", + "--gc-interval-seconds", "21600", + "--gc-low-diskspace-threshold", "2147483648", + "--cache-bucket-limit-overwrites", +) + + +def run(cmd: list[str | Path]) -> None: + try: + subprocess.run(cmd, check=True) + except FileNotFoundError: + sys.exit(f"error: executable not found: {cmd[0]}") + except subprocess.CalledProcessError as e: + sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}") + + +def stop_server(label: str, port: int) -> None: + """Stop a zen server. Tolerates failures so it is safe to call from finally blocks.""" + print(f"--------- stopping {label}") + try: + subprocess.run([ZEN_EXE, "down", "--port", str(port)]) + except OSError as e: + print(f"warning: could not stop {label}: {e}", file=sys.stderr) + print() + + +def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None: + print(f"--------- starting {label} {data_dir}") + run([ + ZEN_EXE, "up", "--port", str(port), "--show-console", "--", + f"--data-dir={data_dir}", + *SERVER_ARGS, + *(extra_args or []), + ]) + print() + + +def wipe_or_create(label: str, path: Path) -> None: + if path.exists(): + print(f"--------- cleaning {label} {path}") + run([ZEN_EXE, "wipe", "-y", path]) + else: + print(f"--------- creating {label} {path}") + path.mkdir(parents=True, exist_ok=True) + print() + + +def check_prerequisites() -> None: + if not ZEN_EXE.is_file(): + sys.exit(f"error: zen executable not found: {ZEN_EXE}") + if not ZEN_METADATA_DIR.is_dir(): + sys.exit(f"error: metadata directory not found: {ZEN_METADATA_DIR}") + for build in BUILDS: + metadata = ZEN_METADATA_DIR / f"{build.name}.json" + if not metadata.is_file(): + sys.exit(f"error: metadata file not found: {metadata}") + + +def main() -> None: + global ZEN_EXE + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "positional_path", + nargs="?", + default=None, + type=Path, + metavar="DATA_PATH", + help="root path for all data directories (positional shorthand for --data-path)", + ) + parser.add_argument( + "zen_exe_positional", + nargs="?", + default=None, + type=Path, + metavar="ZEN_EXE_PATH", + help="path to zen executable (positional shorthand for --zen-exe-path)", + ) + parser.add_argument( + "--data-path", + default=Path(Path(__file__).stem + "_datadir"), + type=Path, + metavar="PATH", + help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + ) + parser.add_argument( + "--zen-exe-path", + default=ZEN_EXE, + type=Path, + metavar="PATH", + help=f"path to zen executable (default: {ZEN_EXE})", + ) + args = parser.parse_args() + + data_path = args.positional_path + if data_path is None: + data_path = args.data_path + + ZEN_EXE = args.zen_exe_positional + if ZEN_EXE is None: + ZEN_EXE = args.zen_exe_path + zen_system_dir = data_path / "system" + zen_download_dir = data_path / "Download" + zen_cache_data_dir = data_path / "ZenBuildsCache" + zen_upload_dir = data_path / "Upload" + zen_chunk_cache_path = data_path / "ChunkCache" + + check_prerequisites() + + start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"]) + try: + wipe_or_create("download folder", zen_download_dir) + wipe_or_create("system folder", zen_system_dir) + + for build in BUILDS: + print(f"--------- importing {build.name} build") + run([ + ZEN_EXE, "builds", "download", + "--host", "https://jupiter.devtools.epicgames.com", + "--namespace", "fortnite.oplog", + "--bucket", build.bucket, + "--build-id", build.id, + "--local-path", zen_download_dir / build.name, + f"--zen-cache-host={ZEN_CACHE}", + f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}", + "--verify", + "--system-dir", zen_system_dir, + ]) + print() + + wipe_or_create("upload folder", zen_upload_dir) + + for build in BUILDS: + print(f"--------- exporting {build.name} build") + run([ + ZEN_EXE, "builds", "upload", + "--storage-path", zen_upload_dir, + "--build-id", build.id, + "--local-path", zen_download_dir / build.name, + "--verify", + "--system-dir", zen_system_dir, + "--metadata-path", str(ZEN_METADATA_DIR / f"{build.name}.json"), + "--create-build", + "--chunking-cache-path", zen_chunk_cache_path, + ]) + print() + finally: + stop_server("cache zenserver", ZEN_CACHE_PORT) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_scripts/metadatas/AndroidClient.json b/scripts/test_scripts/metadatas/AndroidClient.json new file mode 100644 index 000000000..378d0454d --- /dev/null +++ b/scripts/test_scripts/metadatas/AndroidClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 AndroidClient", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Android", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/IOSClient.json b/scripts/test_scripts/metadatas/IOSClient.json new file mode 100644 index 000000000..fb0f9a342 --- /dev/null +++ b/scripts/test_scripts/metadatas/IOSClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 IOSClient", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "IOS", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/LinuxServer.json b/scripts/test_scripts/metadatas/LinuxServer.json new file mode 100644 index 000000000..02ae2d970 --- /dev/null +++ b/scripts/test_scripts/metadatas/LinuxServer.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 LinuxServer", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Linux", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/PS4Client.json b/scripts/test_scripts/metadatas/PS4Client.json new file mode 100644 index 000000000..6e49e3e5e --- /dev/null +++ b/scripts/test_scripts/metadatas/PS4Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 PS4Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "PS4", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/Switch2Client.json b/scripts/test_scripts/metadatas/Switch2Client.json new file mode 100644 index 000000000..41732e7bc --- /dev/null +++ b/scripts/test_scripts/metadatas/Switch2Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 Switch2Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Switch2", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/SwitchClient.json b/scripts/test_scripts/metadatas/SwitchClient.json new file mode 100644 index 000000000..49362f23e --- /dev/null +++ b/scripts/test_scripts/metadatas/SwitchClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 SwitchClient", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Switch", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/WindowsClient.json b/scripts/test_scripts/metadatas/WindowsClient.json new file mode 100644 index 000000000..c7af270c2 --- /dev/null +++ b/scripts/test_scripts/metadatas/WindowsClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 Windows Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Windows", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/XB1Client.json b/scripts/test_scripts/metadatas/XB1Client.json new file mode 100644 index 000000000..36fb45801 --- /dev/null +++ b/scripts/test_scripts/metadatas/XB1Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 XB1Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "XB1", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/oplog-import-export-test.py b/scripts/test_scripts/oplog-import-export-test.py new file mode 100644 index 000000000..51593d5a9 --- /dev/null +++ b/scripts/test_scripts/oplog-import-export-test.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Test script for oplog import/export operations.""" + +from __future__ import annotations + +import argparse +import platform +import subprocess +import sys +from pathlib import Path +from typing import NamedTuple + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" + + +class Build(NamedTuple): + name: str + bucket: str + id: str + + +BUILDS = [ + Build("XB1Client", "fortnitegame.oplog.fortnite-main.xb1client", "09a75f7f3b7517653dcdaaa4"), + Build("WindowsClient", "fortnitegame.oplog.fortnite-main.windowsclient", "09a75d977ef944ecfd0eddfd"), + Build("SwitchClient", "fortnitegame.oplog.fortnite-main.switchclient", "09a74d03b3598ec94cfd2644"), + Build("XSXClient", "fortnitegame.oplog.fortnite-main.xsxclient", "09a76c2bbd6cd78f4d40d9ea"), + Build("Switch2Client", "fortnitegame.oplog.fortnite-main.switch2client", "09a7686b3d9faa78fb24a38f"), + Build("PS4Client", "fortnitegame.oplog.fortnite-main.ps4client", "09a75b72d1c260ed26020140"), + Build("LinuxServer", "fortnitegame.oplog.fortnite-main.linuxserver", "09a747f5e0ee83a04be013e6"), + Build("IOSClient", "fortnitegame.oplog.fortnite-main.iosclient", "09a75f677e883325a209148c"), + Build("Android_ASTCClient", "fortnitegame.oplog.fortnite-main.android_astcclient", "09a7422c08c6f37becc7d37f"), +] + +ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") + +ZEN_PORT = 8558 +ZEN_CACHE_PORT = 8559 +ZEN_CACHE = f"http://127.0.0.1:{ZEN_CACHE_PORT}" +ZEN_CACHE_POPULATE = "true" +ZEN_PARTIAL_REQUEST_MODE = "true" + +SERVER_ARGS: tuple[str, ...] = ( + "--http", "asio", + "--gc-cache-duration-seconds", "1209600", + "--gc-interval-seconds", "21600", + "--gc-low-diskspace-threshold", "2147483648", + "--cache-bucket-limit-overwrites", +) + + +def run(cmd: list[str | Path]) -> None: + try: + subprocess.run(cmd, check=True) + except FileNotFoundError: + sys.exit(f"error: executable not found: {cmd[0]}") + except subprocess.CalledProcessError as e: + sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}") + + +def stop_server(label: str, port: int) -> None: + """Stop a zen server. Tolerates failures so it is safe to call from finally blocks.""" + print(f"--------- stopping {label}") + try: + subprocess.run([ZEN_EXE, "down", "--port", str(port)]) + except OSError as e: + print(f"warning: could not stop {label}: {e}", file=sys.stderr) + print() + + +def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None: + print(f"--------- starting {label} {data_dir}") + run([ + ZEN_EXE, "up", "--port", str(port), "--show-console", "--", + f"--data-dir={data_dir}", + *SERVER_ARGS, + *(extra_args or []), + ]) + print() + + +def wipe_or_create(label: str, path: Path) -> None: + if path.exists(): + print(f"--------- cleaning {label} {path}") + run([ZEN_EXE, "wipe", "-y", path]) + else: + print(f"--------- creating {label} {path}") + path.mkdir(parents=True, exist_ok=True) + print() + + +def check_prerequisites() -> None: + if not ZEN_EXE.is_file(): + sys.exit(f"error: zen executable not found: {ZEN_EXE}") + + +def setup_project(port: int) -> None: + """Create the FortniteGame project and all oplogs on the server at the given port.""" + print("--------- creating FortniteGame project") + run([ZEN_EXE, "project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update"]) + print() + + for build in BUILDS: + print(f"--------- creating {build.name} oplog") + run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build.name, "--force-update"]) + print() + + +def main() -> None: + global ZEN_EXE + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "positional_path", + nargs="?", + default=None, + type=Path, + metavar="DATA_PATH", + help="root path for all data directories (positional shorthand for --data-path)", + ) + parser.add_argument( + "zen_exe_positional", + nargs="?", + default=None, + type=Path, + metavar="ZEN_EXE_PATH", + help="path to zen executable (positional shorthand for --zen-exe-path)", + ) + parser.add_argument( + "--data-path", + default=Path(Path(__file__).stem + "_datadir"), + type=Path, + metavar="PATH", + help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + ) + parser.add_argument( + "--zen-exe-path", + default=ZEN_EXE, + type=Path, + metavar="PATH", + help=f"path to zen executable (default: {ZEN_EXE})", + ) + args = parser.parse_args() + + data_path = args.positional_path + if data_path is None: + data_path = args.data_path + + ZEN_EXE = args.zen_exe_positional + if ZEN_EXE is None: + ZEN_EXE = args.zen_exe_path + zen_data_dir = data_path / "DDC" / "OplogsZen" + zen_cache_data_dir = data_path / "DDC" / "ZenBuildsCache" + zen_import_data_dir = data_path / "DDC" / "OplogsZenImport" + export_dir = data_path / "Export" / "FortniteGame" + + check_prerequisites() + + start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"]) + try: + wipe_or_create("zenserver data", zen_data_dir) + start_server("zenserver", zen_data_dir, ZEN_PORT) + try: + setup_project(ZEN_PORT) + + for build in BUILDS: + print(f"--------- importing {build.name} oplog") + run([ + ZEN_EXE, "oplog-import", + f"--hosturl=127.0.0.1:{ZEN_PORT}", + "FortniteGame", build.name, + "--clean", + "--builds", "https://jupiter.devtools.epicgames.com", + "--namespace", "fortnite.oplog", + "--bucket", build.bucket, + "--builds-id", build.id, + f"--zen-cache-host={ZEN_CACHE}", + f"--zen-cache-upload={ZEN_CACHE_POPULATE}", + f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}", + ]) + print() + + print(f"--------- validating {build.name} oplog") + run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + print() + + wipe_or_create("export folder", export_dir) + + for build in BUILDS: + print(f"--------- exporting {build.name} oplog") + run([ + ZEN_EXE, "oplog-export", + f"--hosturl=127.0.0.1:{ZEN_PORT}", + "FortniteGame", build.name, + "--file", export_dir, + "--forcetempblocks", + ]) + print() + finally: + stop_server("zenserver", ZEN_PORT) + + wipe_or_create("alternate zenserver data", zen_import_data_dir) + start_server("import zenserver", zen_import_data_dir, ZEN_PORT) + try: + setup_project(ZEN_PORT) + + for build in BUILDS: + print(f"--------- importing {build.name} oplog") + run([ + ZEN_EXE, "oplog-import", + f"--hosturl=127.0.0.1:{ZEN_PORT}", + "FortniteGame", build.name, + "--file", export_dir, + ]) + print() + + print(f"--------- validating {build.name} oplog") + run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + print() + finally: + stop_server("alternative zenserver", ZEN_PORT) + finally: + stop_server("cache zenserver", ZEN_CACHE_PORT) + + +if __name__ == "__main__": + main() -- cgit v1.2.3 From 9f33eaa7b7f246d68ea539f32cd74ecf8dfd5790 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Mar 2026 22:20:52 +0100 Subject: =?UTF-8?q?updated=20chunk=E2=80=93block=20analyser=20(#818)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * create oplogs as they are imported * Improved logic for partial block analisys * unit tests for ChunkBlockAnalyser --- scripts/test_scripts/oplog-import-export-test.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/test_scripts/oplog-import-export-test.py b/scripts/test_scripts/oplog-import-export-test.py index 51593d5a9..b2a5ece6c 100644 --- a/scripts/test_scripts/oplog-import-export-test.py +++ b/scripts/test_scripts/oplog-import-export-test.py @@ -96,15 +96,17 @@ def check_prerequisites() -> None: def setup_project(port: int) -> None: - """Create the FortniteGame project and all oplogs on the server at the given port.""" + """Create the FortniteGame project on the server at the given port.""" print("--------- creating FortniteGame project") run([ZEN_EXE, "project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update"]) print() - for build in BUILDS: - print(f"--------- creating {build.name} oplog") - run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build.name, "--force-update"]) - print() + +def setup_oplog(port: int, build_name: str) -> None: + """Create the oplog in the FortniteGame project on the server at the given port.""" + print(f"--------- creating {build_name} oplog") + run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build_name, "--force-update"]) + print() def main() -> None: @@ -165,6 +167,8 @@ def main() -> None: setup_project(ZEN_PORT) for build in BUILDS: + setup_oplog(ZEN_PORT, build.name) + print(f"--------- importing {build.name} oplog") run([ ZEN_EXE, "oplog-import", @@ -206,6 +210,8 @@ def main() -> None: setup_project(ZEN_PORT) for build in BUILDS: + setup_oplog(ZEN_PORT, build.name) + print(f"--------- importing {build.name} oplog") run([ ZEN_EXE, "oplog-import", -- cgit v1.2.3 From d0a07e555577dcd4a8f55f1b45d9e8e4e6366ab7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 10 Mar 2026 17:27:26 +0100 Subject: HttpClient using libcurl, Unix Sockets for HTTP. HTTPS support (#770) The main goal of this change is to eliminate the cpr back-end altogether and replace it with the curl implementation. I would expect to drop cpr as soon as we feel happy with the libcurl back-end. That would leave us with a direct dependency on libcurl only, and cpr can be eliminated as a dependency. ### HttpClient Backend Overhaul - Implemented a new **libcurl-based HttpClient** backend (`httpclientcurl.cpp`, ~2000 lines) as an alternative to the cpr-based one - Made HttpClient backend **configurable at runtime** via constructor arguments and `-httpclient=...` CLI option (for zen, zenserver, and tests) - Extended HttpClient test suite to cover multipart/content-range scenarios ### Unix Domain Socket Support - Added Unix domain socket support to **httpasio** (server side) - Added Unix domain socket support to **HttpClient** - Added Unix domain socket support to **HttpWsClient** (WebSocket client) - Templatized `HttpServerConnectionT` and `WsAsioConnectionT` to handle TCP, Unix, and SSL sockets uniformly via `if constexpr` dispatch ### HTTPS Support - Added **preliminary HTTPS support to httpasio** (for Mac/Linux via OpenSSL) - Added **basic HTTPS support for http.sys** (Windows) - Implemented HTTPS test for httpasio - Split `InitializeServer` into smaller sub-functions for http.sys ### Other Notable Changes - Improved **zenhttp-test stability** with dynamic port allocation - Enhanced port retry logic in http.sys (handles ERROR_ACCESS_DENIED) - Fatal signal/exception handlers for backtrace generation in tests - Added `zen bench http` subcommand to exercise network + HTTP client/server communication stack --- .../test_scripts/builds-download-upload-test.py | 70 +++++++++++++++------- 1 file changed, 48 insertions(+), 22 deletions(-) (limited to 'scripts') diff --git a/scripts/test_scripts/builds-download-upload-test.py b/scripts/test_scripts/builds-download-upload-test.py index f03528d98..e4fee7cb8 100644 --- a/scripts/test_scripts/builds-download-upload-test.py +++ b/scripts/test_scripts/builds-download-upload-test.py @@ -49,7 +49,13 @@ SERVER_ARGS: tuple[str, ...] = ( ) +def zen_cmd(*args: str | Path, extra_zen_args: list[str] | None = None) -> list[str | Path]: + """Build a zen CLI command list, inserting extra_zen_args before subcommands.""" + return [ZEN_EXE, *(extra_zen_args or []), *args] + + def run(cmd: list[str | Path]) -> None: + print(f" > {' '.join(str(x) for x in cmd)}") try: subprocess.run(cmd, check=True) except FileNotFoundError: @@ -58,31 +64,35 @@ def run(cmd: list[str | Path]) -> None: sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}") -def stop_server(label: str, port: int) -> None: +def stop_server(label: str, port: int, extra_zen_args: list[str] | None = None) -> None: """Stop a zen server. Tolerates failures so it is safe to call from finally blocks.""" print(f"--------- stopping {label}") + cmd = zen_cmd("down", "--port", str(port), extra_zen_args=extra_zen_args) + print(f" > {' '.join(str(x) for x in cmd)}") try: - subprocess.run([ZEN_EXE, "down", "--port", str(port)]) + subprocess.run(cmd) except OSError as e: print(f"warning: could not stop {label}: {e}", file=sys.stderr) print() -def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None: +def start_server(label: str, data_dir: Path, port: int, extra_zen_args: list[str] | None = None, + extra_server_args: list[str] | None = None) -> None: print(f"--------- starting {label} {data_dir}") - run([ - ZEN_EXE, "up", "--port", str(port), "--show-console", "--", + run(zen_cmd( + "up", "--port", str(port), "--show-console", "--", f"--data-dir={data_dir}", *SERVER_ARGS, - *(extra_args or []), - ]) + *(extra_server_args or []), + extra_zen_args=extra_zen_args, + )) print() -def wipe_or_create(label: str, path: Path) -> None: +def wipe_or_create(label: str, path: Path, extra_zen_args: list[str] | None = None) -> None: if path.exists(): print(f"--------- cleaning {label} {path}") - run([ZEN_EXE, "wipe", "-y", path]) + run(zen_cmd("wipe", "-y", path, extra_zen_args=extra_zen_args)) else: print(f"--------- creating {label} {path}") path.mkdir(parents=True, exist_ok=True) @@ -103,7 +113,20 @@ def check_prerequisites() -> None: def main() -> None: global ZEN_EXE - parser = argparse.ArgumentParser(description=__doc__) + # Split on '--' to separate script args from extra zen CLI args + script_argv: list[str] = [] + extra_zen_args: list[str] = [] + if "--" in sys.argv[1:]: + sep = sys.argv.index("--", 1) + script_argv = sys.argv[1:sep] + extra_zen_args = sys.argv[sep + 1:] + else: + script_argv = sys.argv[1:] + + parser = argparse.ArgumentParser( + description=__doc__, + epilog="Any arguments after '--' are forwarded to every zen CLI invocation.", + ) parser.add_argument( "positional_path", nargs="?", @@ -134,7 +157,7 @@ def main() -> None: metavar="PATH", help=f"path to zen executable (default: {ZEN_EXE})", ) - args = parser.parse_args() + args = parser.parse_args(script_argv) data_path = args.positional_path if data_path is None: @@ -151,15 +174,16 @@ def main() -> None: check_prerequisites() - start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"]) + start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, + extra_zen_args=extra_zen_args, extra_server_args=["--buildstore-enabled"]) try: - wipe_or_create("download folder", zen_download_dir) - wipe_or_create("system folder", zen_system_dir) + wipe_or_create("download folder", zen_download_dir, extra_zen_args) + wipe_or_create("system folder", zen_system_dir, extra_zen_args) for build in BUILDS: print(f"--------- importing {build.name} build") - run([ - ZEN_EXE, "builds", "download", + run(zen_cmd( + "builds", "download", "--host", "https://jupiter.devtools.epicgames.com", "--namespace", "fortnite.oplog", "--bucket", build.bucket, @@ -169,15 +193,16 @@ def main() -> None: f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}", "--verify", "--system-dir", zen_system_dir, - ]) + extra_zen_args=extra_zen_args, + )) print() - wipe_or_create("upload folder", zen_upload_dir) + wipe_or_create("upload folder", zen_upload_dir, extra_zen_args) for build in BUILDS: print(f"--------- exporting {build.name} build") - run([ - ZEN_EXE, "builds", "upload", + run(zen_cmd( + "builds", "upload", "--storage-path", zen_upload_dir, "--build-id", build.id, "--local-path", zen_download_dir / build.name, @@ -186,10 +211,11 @@ def main() -> None: "--metadata-path", str(ZEN_METADATA_DIR / f"{build.name}.json"), "--create-build", "--chunking-cache-path", zen_chunk_cache_path, - ]) + extra_zen_args=extra_zen_args, + )) print() finally: - stop_server("cache zenserver", ZEN_CACHE_PORT) + stop_server("cache zenserver", ZEN_CACHE_PORT, extra_zen_args) if __name__ == "__main__": -- cgit v1.2.3 From 43b3823f2eea918162f53dd5409dcaa9dbd2fb02 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 11 Mar 2026 16:55:39 +0100 Subject: fix OidcToken casing (#826) fixes issue with zip not finding the OidcToken file --- scripts/bundle.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/bundle.lua b/scripts/bundle.lua index 6f540c5b8..74b961d02 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -279,7 +279,7 @@ local function main_mac(signidentity) "build/macosx/universal/release/zenserver", "build/macosx/universal/release/zen", "build/macosx/universal/release/crashpad_handler", - "build/macosx/x86_64/release/Oidctoken") + "build/macosx/x86_64/release/OidcToken") end -------------------------------------------------------------------------------- @@ -291,7 +291,7 @@ local function main_linux() "build/linux/x86_64/release/zenserver", "build/linux/x86_64/release/zen", "build/linux/x86_64/release/crashpad_handler", - "build/linux/x86_64/release/Oidctoken") + "build/linux/x86_64/release/OidcToken") end -------------------------------------------------------------------------------- -- cgit v1.2.3 From be693a0b5dbd70271da04989a664c8c3b6919ae1 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 Mar 2026 14:36:26 +0100 Subject: fix casing of windows OidcToken.exe (#830) --- scripts/bundle.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/bundle.lua b/scripts/bundle.lua index 74b961d02..6f4552890 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -200,7 +200,7 @@ local function main_windows(signidentity) "build/windows/x64/release/zen.exe", "build/windows/x64/release/zen.pdb", "build/windows/x64/release/crashpad_handler.exe", - "build/windows/x64/release/oidctoken.exe") + "build/windows/x64/release/OidcToken.exe") end -------------------------------------------------------------------------------- -- cgit v1.2.3 From 162d7412405e78198ede361a8fbae8dc8b82278a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 13 Mar 2026 22:30:32 +0100 Subject: Made CPR optional, html generated at build time (#840) - Fix potential crash on startup caused by logging macros being invoked before the logging system is initialized (null logger dereference in `ZenServerState::Sweep()`). `LoggerRef::ShouldLog` now guards against a null logger pointer. - Make CPR an optional dependency (`--zencpr` build option, enabled by default) so builds can proceed without it - Make zenvfs Windows-only (platform-specific target) - Generate the frontend zip at build time from source HTML files instead of checking in a binary blob which would accumulate with every single update --- scripts/updatefrontend.lua | 111 --------------------------------------------- 1 file changed, 111 deletions(-) delete mode 100644 scripts/updatefrontend.lua (limited to 'scripts') diff --git a/scripts/updatefrontend.lua b/scripts/updatefrontend.lua deleted file mode 100644 index ab37819d7..000000000 --- a/scripts/updatefrontend.lua +++ /dev/null @@ -1,111 +0,0 @@ --- Copyright Epic Games, Inc. All Rights Reserved. - --------------------------------------------------------------------------------- -local function _exec(cmd, ...) - local args = {} - for _, arg in pairs({...}) do - if arg then - table.insert(args, arg) - end - end - - print("--", cmd, table.unpack(args)) - local ret = os.execv(cmd, args) - print() - return ret -end - --------------------------------------------------------------------------------- -local function _zip(store_only, zip_path, ...) - -- Here's the rules; if len(...) is 1 and it is a dir then create a zip with - -- archive paths like this; - -- - -- glob(foo/bar/**) -> foo/bar/abc, foo/bar/dir/123 -> zip(abc, dir/123) - -- - -- Otherwise assume ... is file paths and add without leading directories; - -- - -- foo/abc, bar/123 -> zip(abc, 123) - - zip_path = path.absolute(zip_path) - os.tryrm(zip_path) - - local inputs = {...} - - local source_dir = nil - if #inputs == 1 and os.isdir(inputs[1]) then - source_dir = inputs[1] - end - - import("detect.tools.find_7z") - local cmd_7z = find_7z() - if cmd_7z then - input_paths = {} - if source_dir then - -- Suffixing a directory path with a "/." will have 7z set the path - -- for archived files relative to that directory. - input_paths = { path.join(source_dir, ".") } - else - for _, input_path in pairs(inputs) do - -- If there is a "/./" anywhere in file paths then 7z drops all - -- directory information and just archives the file by name - input_path = path.relative(input_path, ".") - if input_path:sub(2,2) ~= ":" then - input_path = "./"..input_path - end - table.insert(input_paths, input_path) - end - end - - compression_level = "-mx1" - if store_only then - compression_level = "-mx0" - end - - local ret = _exec(cmd_7z, "a", compression_level, zip_path, table.unpack(input_paths)) - if ret > 0 then - raise("Received error from 7z") - end - return - end - - print("7z not found, falling back to zip") - - import("detect.tools.find_zip") - zip_cmd = find_zip() - if zip_cmd then - local input_paths = inputs - local cwd = os.curdir() - if source_dir then - os.cd(source_dir) - input_paths = { "." } - end - - compression_level = "-1" - if store_only then - compression_level = "-0" - end - - local strip_leading_path = nil - if not source_dir then - strip_leading_path = "--junk-paths" - end - - local ret = _exec(zip_cmd, "-r", compression_level, strip_leading_path, zip_path, table.unpack(input_paths)) - if ret > 0 then - raise("Received error from zip") - end - - os.cd(cwd) - return - end - print("zip not found") - - raise("Unable to find a suitable zip tool") -end - --------------------------------------------------------------------------------- -function main() - local zip_path = "src/zenserver/frontend/html.zip" - local content_dir = "src/zenserver/frontend/html/" - _zip(true, zip_path, content_dir) -end -- cgit v1.2.3 From 20723233f276cbcfd3cd82f628bc2075a33ae435 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 14 Mar 2026 14:15:06 +0100 Subject: add buildid updates to oplog and builds test scripts (#838) * updated test scripts * remove some hardcoded assumptions --- .../test_scripts/builds-download-upload-test.py | 74 ++++++--- .../builds-download-upload-update-build-ids.py | 150 +++++++++++++++++ scripts/test_scripts/oplog-import-export-test.py | 177 ++++++++++++++------- scripts/test_scripts/oplog-update-build-ids.py | 151 ++++++++++++++++++ 4 files changed, 478 insertions(+), 74 deletions(-) create mode 100644 scripts/test_scripts/builds-download-upload-update-build-ids.py create mode 100644 scripts/test_scripts/oplog-update-build-ids.py (limited to 'scripts') diff --git a/scripts/test_scripts/builds-download-upload-test.py b/scripts/test_scripts/builds-download-upload-test.py index e4fee7cb8..8ff5245c1 100644 --- a/scripts/test_scripts/builds-download-upload-test.py +++ b/scripts/test_scripts/builds-download-upload-test.py @@ -4,6 +4,8 @@ from __future__ import annotations import argparse +import json +import os import platform import subprocess import sys @@ -15,22 +17,51 @@ _ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() _EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" +def _cache_dir() -> Path: + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return base / "Temp" / "zen" + elif sys.platform == "darwin": + return Path.home() / "Library" / "Caches" / "zen" + else: + base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return base / "zen" + + +_BUILD_IDS_PATH = _cache_dir() / "builds-download-upload-build-ids.json" + + class Build(NamedTuple): name: str bucket: str id: str -BUILDS = [ - Build("XB1Client", "fortnitegame.staged-build.fortnite-main.xb1-client", "09a7616c1a388dfe6056aa57"), - Build("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client", "09a762c81e2cf213142d0ce5"), - Build("SwitchClient", "fortnitegame.staged-build.fortnite-main.switch-client", "09a75bf9c3ce75bce09f644f"), - Build("LinuxServer", "fortnitegame.staged-build.fortnite-main.linux-server", "09a750ac155eb3e3b62e87e0"), - Build("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client", "09a78f3df07b289691ec5710"), - Build("PS4Client", "fortnitegame.staged-build.fortnite-main.ps4-client", "09a76ea92ad301d4724fafad"), - Build("IOSClient", "fortnitegame.staged-build.fortnite-main.ios-client", "09a7816fa26c23362fef0c5d"), - Build("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client", "09a76725f1620d62c6be06e4"), -] +def load_builds() -> tuple[str, list[Build]]: + if not _BUILD_IDS_PATH.exists(): + print(f"Build IDs file not found: {_BUILD_IDS_PATH}") + answer = input("Run builds-download-upload-update-build-ids.py now to populate it? [y/N] ").strip().lower() + if answer == "y": + update_script = Path(__file__).parent / "builds-download-upload-update-build-ids.py" + subprocess.run([sys.executable, str(update_script)], check=True) + else: + sys.exit("Aborted. Run scripts/test_scripts/builds-download-upload-update-build-ids.py to populate it.") + with _BUILD_IDS_PATH.open() as f: + data: dict = json.load(f) + namespace = data.get("namespace", "") + if not namespace: + sys.exit(f"error: {_BUILD_IDS_PATH} is missing 'namespace'") + builds = [] + for name, entry in data.get("builds", {}).items(): + bucket = entry.get("bucket", "") + build_id = entry.get("buildId", "") + if not bucket or not build_id: + sys.exit(f"error: entry '{name}' in {_BUILD_IDS_PATH} is missing 'bucket' or 'buildId'") + builds.append(Build(name, bucket, build_id)) + if not builds: + sys.exit(f"error: {_BUILD_IDS_PATH} contains no builds") + return namespace, builds + ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") ZEN_METADATA_DIR: Path = Path(__file__).resolve().parent / "metadatas" @@ -99,12 +130,12 @@ def wipe_or_create(label: str, path: Path, extra_zen_args: list[str] | None = No print() -def check_prerequisites() -> None: +def check_prerequisites(builds: list[Build]) -> None: if not ZEN_EXE.is_file(): sys.exit(f"error: zen executable not found: {ZEN_EXE}") if not ZEN_METADATA_DIR.is_dir(): sys.exit(f"error: metadata directory not found: {ZEN_METADATA_DIR}") - for build in BUILDS: + for build in builds: metadata = ZEN_METADATA_DIR / f"{build.name}.json" if not metadata.is_file(): sys.exit(f"error: metadata file not found: {metadata}") @@ -145,10 +176,10 @@ def main() -> None: ) parser.add_argument( "--data-path", - default=Path(Path(__file__).stem + "_datadir"), + default=None, type=Path, metavar="PATH", - help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + help="root path for all data directories", ) parser.add_argument( "--zen-exe-path", @@ -162,17 +193,24 @@ def main() -> None: data_path = args.positional_path if data_path is None: data_path = args.data_path + if data_path is None: + print("WARNING: This script may require up to 1TB of free disk space.") + raw = input("Enter root path for all data directories: ").strip() + if not raw: + sys.exit("error: data path is required") + data_path = Path(raw) ZEN_EXE = args.zen_exe_positional if ZEN_EXE is None: ZEN_EXE = args.zen_exe_path + namespace, builds = load_builds() zen_system_dir = data_path / "system" zen_download_dir = data_path / "Download" zen_cache_data_dir = data_path / "ZenBuildsCache" zen_upload_dir = data_path / "Upload" zen_chunk_cache_path = data_path / "ChunkCache" - check_prerequisites() + check_prerequisites(builds) start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, extra_zen_args=extra_zen_args, extra_server_args=["--buildstore-enabled"]) @@ -180,12 +218,12 @@ def main() -> None: wipe_or_create("download folder", zen_download_dir, extra_zen_args) wipe_or_create("system folder", zen_system_dir, extra_zen_args) - for build in BUILDS: + for build in builds: print(f"--------- importing {build.name} build") run(zen_cmd( "builds", "download", "--host", "https://jupiter.devtools.epicgames.com", - "--namespace", "fortnite.oplog", + "--namespace", namespace, "--bucket", build.bucket, "--build-id", build.id, "--local-path", zen_download_dir / build.name, @@ -199,7 +237,7 @@ def main() -> None: wipe_or_create("upload folder", zen_upload_dir, extra_zen_args) - for build in BUILDS: + for build in builds: print(f"--------- exporting {build.name} build") run(zen_cmd( "builds", "upload", diff --git a/scripts/test_scripts/builds-download-upload-update-build-ids.py b/scripts/test_scripts/builds-download-upload-update-build-ids.py new file mode 100644 index 000000000..2a63aa44d --- /dev/null +++ b/scripts/test_scripts/builds-download-upload-update-build-ids.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Update builds-download-upload-build-ids.json with build IDs at the highest common changelist across all buckets.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +import tempfile +from pathlib import Path + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" +_DEFAULT_ZEN = Path(f"build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") + + +def _cache_dir() -> Path: + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return base / "Temp" / "zen" + elif sys.platform == "darwin": + return Path.home() / "Library" / "Caches" / "zen" + else: + base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return base / "zen" + + +_OUTPUT_PATH = _cache_dir() / "builds-download-upload-build-ids.json" + +# Maps build name -> Jupiter bucket +_BUILDS: list[tuple[str, str]] = [ + ("XB1Client", "fortnitegame.staged-build.fortnite-main.xb1-client"), + ("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client"), + ("SwitchClient", "fortnitegame.staged-build.fortnite-main.switch-client"), + ("LinuxServer", "fortnitegame.staged-build.fortnite-main.linux-server"), + ("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client"), + ("PS4Client", "fortnitegame.staged-build.fortnite-main.ps4-client"), + ("PS5Client", "fortnitegame.staged-build.fortnite-main.ps5-client"), + ("IOSClient", "fortnitegame.staged-build.fortnite-main.ios-client"), + ("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client"), +] + + +def list_builds_for_bucket(zen: str, host: str, namespace: str, bucket: str) -> list[dict]: + """Run zen builds list for a single bucket and return the results array.""" + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp: + result_path = Path(tmp.name) + + cmd = [ + zen, "builds", "list", + "--namespace", namespace, + "--bucket", bucket, + "--host", host, + "--result-path", str(result_path), + ] + + try: + subprocess.run(cmd, check=True, capture_output=True) + except FileNotFoundError: + sys.exit(f"error: zen binary not found: {zen}") + except subprocess.CalledProcessError as e: + sys.exit( + f"error: zen builds list failed for bucket '{bucket}' with exit code {e.returncode}\n" + f"stderr: {e.stderr.decode(errors='replace')}" + ) + + with result_path.open() as f: + data = json.load(f) + result_path.unlink(missing_ok=True) + + return data.get("results", []) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Refresh builds-download-upload-build-ids.json with build IDs at the highest changelist present in all buckets." + ) + parser.add_argument("--host", default="https://jupiter.devtools.epicgames.com", help="Jupiter host URL") + parser.add_argument("--zen", default=str(_DEFAULT_ZEN), help="Path to the zen binary") + parser.add_argument("--namespace", default="fortnite.oplog", help="Builds storage namespace") + args = parser.parse_args() + + # For each bucket, fetch results and build a changelist -> buildId map. + # bucket_cl_map[bucket] = { changelist_int: buildId_str, ... } + bucket_cl_map: dict[str, dict[int, str]] = {} + + for name, bucket in _BUILDS: + print(f"Querying {name} ({bucket}) ...") + results = list_builds_for_bucket(args.zen, args.host, args.namespace, bucket) + if not results: + sys.exit(f"error: no results for bucket '{bucket}' (build '{name}')") + + cl_map: dict[int, str] = {} + for entry in results: + build_id = entry.get("buildId", "") + metadata = entry.get("metadata") or {} + cl = metadata.get("commit") + if build_id and cl is not None: + # Keep first occurrence (most recent) per changelist + if cl not in cl_map: + cl_map[int(cl)] = build_id + + if not cl_map: + sys.exit( + f"error: bucket '{bucket}' (build '{name}') returned {len(results)} entries " + "but none had both buildId and changelist in metadata" + ) + + print(f" {len(cl_map)} distinct changelists, latest CL {max(cl_map)}") + bucket_cl_map[bucket] = cl_map + + # Find the highest changelist present in every bucket's result set. + common_cls = set(next(iter(bucket_cl_map.values())).keys()) + for bucket, cl_map in bucket_cl_map.items(): + common_cls &= set(cl_map.keys()) + + if not common_cls: + sys.exit( + "error: no changelist is present in all buckets.\n" + "Per-bucket CL ranges:\n" + + "\n".join( + f" {name} ({bucket}): {min(bucket_cl_map[bucket])} – {max(bucket_cl_map[bucket])}" + for name, bucket in _BUILDS + ) + ) + + best_cl = max(common_cls) + print(f"\nHighest common changelist: {best_cl}") + + build_ids: dict[str, dict[str, str]] = {} + for name, bucket in _BUILDS: + build_id = bucket_cl_map[bucket][best_cl] + build_ids[name] = {"bucket": bucket, "buildId": build_id} + print(f" {name}: {build_id}") + + output = {"namespace": args.namespace, "builds": build_ids} + _OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + with _OUTPUT_PATH.open("w") as f: + json.dump(output, f, indent=2) + f.write("\n") + + print(f"\nWrote {_OUTPUT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_scripts/oplog-import-export-test.py b/scripts/test_scripts/oplog-import-export-test.py index b2a5ece6c..f913a7351 100644 --- a/scripts/test_scripts/oplog-import-export-test.py +++ b/scripts/test_scripts/oplog-import-export-test.py @@ -4,6 +4,8 @@ from __future__ import annotations import argparse +import json +import os import platform import subprocess import sys @@ -15,23 +17,51 @@ _ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() _EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" +def _cache_dir() -> Path: + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return base / "Temp" / "zen" + elif sys.platform == "darwin": + return Path.home() / "Library" / "Caches" / "zen" + else: + base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return base / "zen" + + +_BUILD_IDS_PATH = _cache_dir() / "oplog-import-export-build-ids.json" + + class Build(NamedTuple): name: str bucket: str id: str -BUILDS = [ - Build("XB1Client", "fortnitegame.oplog.fortnite-main.xb1client", "09a75f7f3b7517653dcdaaa4"), - Build("WindowsClient", "fortnitegame.oplog.fortnite-main.windowsclient", "09a75d977ef944ecfd0eddfd"), - Build("SwitchClient", "fortnitegame.oplog.fortnite-main.switchclient", "09a74d03b3598ec94cfd2644"), - Build("XSXClient", "fortnitegame.oplog.fortnite-main.xsxclient", "09a76c2bbd6cd78f4d40d9ea"), - Build("Switch2Client", "fortnitegame.oplog.fortnite-main.switch2client", "09a7686b3d9faa78fb24a38f"), - Build("PS4Client", "fortnitegame.oplog.fortnite-main.ps4client", "09a75b72d1c260ed26020140"), - Build("LinuxServer", "fortnitegame.oplog.fortnite-main.linuxserver", "09a747f5e0ee83a04be013e6"), - Build("IOSClient", "fortnitegame.oplog.fortnite-main.iosclient", "09a75f677e883325a209148c"), - Build("Android_ASTCClient", "fortnitegame.oplog.fortnite-main.android_astcclient", "09a7422c08c6f37becc7d37f"), -] +def load_builds() -> tuple[str, list[Build]]: + if not _BUILD_IDS_PATH.exists(): + print(f"Build IDs file not found: {_BUILD_IDS_PATH}") + answer = input("Run oplog-update-build-ids.py now to populate it? [y/N] ").strip().lower() + if answer == "y": + update_script = Path(__file__).parent / "oplog-update-build-ids.py" + subprocess.run([sys.executable, str(update_script)], check=True) + else: + sys.exit("Aborted. Run scripts/test_scripts/oplog-update-build-ids.py to populate it.") + with _BUILD_IDS_PATH.open() as f: + data: dict = json.load(f) + namespace = data.get("namespace", "") + if not namespace: + sys.exit(f"error: {_BUILD_IDS_PATH} is missing 'namespace'") + builds = [] + for name, entry in data.get("builds", {}).items(): + bucket = entry.get("bucket", "") + build_id = entry.get("buildId", "") + if not bucket or not build_id: + sys.exit(f"error: entry '{name}' in {_BUILD_IDS_PATH} is missing 'bucket' or 'buildId'") + builds.append(Build(name, bucket, build_id)) + if not builds: + sys.exit(f"error: {_BUILD_IDS_PATH} contains no builds") + return namespace, builds + ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") @@ -50,6 +80,11 @@ SERVER_ARGS: tuple[str, ...] = ( ) +def zen_cmd(*args: str | Path, extra_zen_args: list[str] | None = None) -> list[str | Path]: + """Build a zen CLI command list, inserting extra_zen_args before subcommands.""" + return [ZEN_EXE, *(extra_zen_args or []), *args] + + def run(cmd: list[str | Path]) -> None: try: subprocess.run(cmd, check=True) @@ -59,31 +94,33 @@ def run(cmd: list[str | Path]) -> None: sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}") -def stop_server(label: str, port: int) -> None: +def stop_server(label: str, port: int, extra_zen_args: list[str] | None = None) -> None: """Stop a zen server. Tolerates failures so it is safe to call from finally blocks.""" print(f"--------- stopping {label}") try: - subprocess.run([ZEN_EXE, "down", "--port", str(port)]) + subprocess.run(zen_cmd("down", "--port", str(port), extra_zen_args=extra_zen_args)) except OSError as e: print(f"warning: could not stop {label}: {e}", file=sys.stderr) print() -def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None: +def start_server(label: str, data_dir: Path, port: int, extra_zen_args: list[str] | None = None, + extra_server_args: list[str] | None = None) -> None: print(f"--------- starting {label} {data_dir}") - run([ - ZEN_EXE, "up", "--port", str(port), "--show-console", "--", + run(zen_cmd( + "up", "--port", str(port), "--show-console", "--", f"--data-dir={data_dir}", *SERVER_ARGS, - *(extra_args or []), - ]) + *(extra_server_args or []), + extra_zen_args=extra_zen_args, + )) print() -def wipe_or_create(label: str, path: Path) -> None: +def wipe_or_create(label: str, path: Path, extra_zen_args: list[str] | None = None) -> None: if path.exists(): print(f"--------- cleaning {label} {path}") - run([ZEN_EXE, "wipe", "-y", path]) + run(zen_cmd("wipe", "-y", path, extra_zen_args=extra_zen_args)) else: print(f"--------- creating {label} {path}") path.mkdir(parents=True, exist_ok=True) @@ -95,24 +132,39 @@ def check_prerequisites() -> None: sys.exit(f"error: zen executable not found: {ZEN_EXE}") -def setup_project(port: int) -> None: +def setup_project(port: int, extra_zen_args: list[str] | None = None) -> None: """Create the FortniteGame project on the server at the given port.""" print("--------- creating FortniteGame project") - run([ZEN_EXE, "project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update"]) + run(zen_cmd("project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update", + extra_zen_args=extra_zen_args)) print() -def setup_oplog(port: int, build_name: str) -> None: +def setup_oplog(port: int, build_name: str, extra_zen_args: list[str] | None = None) -> None: """Create the oplog in the FortniteGame project on the server at the given port.""" print(f"--------- creating {build_name} oplog") - run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build_name, "--force-update"]) + run(zen_cmd("oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build_name, "--force-update", + extra_zen_args=extra_zen_args)) print() def main() -> None: global ZEN_EXE - parser = argparse.ArgumentParser(description=__doc__) + # Split on '--' to separate script args from extra zen CLI args + script_argv: list[str] = [] + extra_zen_args: list[str] = [] + if "--" in sys.argv[1:]: + sep = sys.argv.index("--", 1) + script_argv = sys.argv[1:sep] + extra_zen_args = sys.argv[sep + 1:] + else: + script_argv = sys.argv[1:] + + parser = argparse.ArgumentParser( + description=__doc__, + epilog="Any arguments after '--' are forwarded to every zen CLI invocation.", + ) parser.add_argument( "positional_path", nargs="?", @@ -131,10 +183,10 @@ def main() -> None: ) parser.add_argument( "--data-path", - default=Path(Path(__file__).stem + "_datadir"), + default=None, type=Path, metavar="PATH", - help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + help="root path for all data directories", ) parser.add_argument( "--zen-exe-path", @@ -143,15 +195,22 @@ def main() -> None: metavar="PATH", help=f"path to zen executable (default: {ZEN_EXE})", ) - args = parser.parse_args() + args = parser.parse_args(script_argv) data_path = args.positional_path if data_path is None: data_path = args.data_path + if data_path is None: + print("WARNING: This script may require up to 1TB of free disk space.") + raw = input("Enter root path for all data directories: ").strip() + if not raw: + sys.exit("error: data path is required") + data_path = Path(raw) ZEN_EXE = args.zen_exe_positional if ZEN_EXE is None: ZEN_EXE = args.zen_exe_path + namespace, builds = load_builds() zen_data_dir = data_path / "DDC" / "OplogsZen" zen_cache_data_dir = data_path / "DDC" / "ZenBuildsCache" zen_import_data_dir = data_path / "DDC" / "OplogsZenImport" @@ -159,75 +218,81 @@ def main() -> None: check_prerequisites() - start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"]) + start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, + extra_zen_args=extra_zen_args, extra_server_args=["--buildstore-enabled"]) try: - wipe_or_create("zenserver data", zen_data_dir) - start_server("zenserver", zen_data_dir, ZEN_PORT) + wipe_or_create("zenserver data", zen_data_dir, extra_zen_args) + start_server("zenserver", zen_data_dir, ZEN_PORT, extra_zen_args=extra_zen_args) try: - setup_project(ZEN_PORT) + setup_project(ZEN_PORT, extra_zen_args) - for build in BUILDS: - setup_oplog(ZEN_PORT, build.name) + for build in builds: + setup_oplog(ZEN_PORT, build.name, extra_zen_args) print(f"--------- importing {build.name} oplog") - run([ - ZEN_EXE, "oplog-import", + run(zen_cmd( + "oplog-import", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, "--clean", "--builds", "https://jupiter.devtools.epicgames.com", - "--namespace", "fortnite.oplog", + "--namespace", namespace, "--bucket", build.bucket, "--builds-id", build.id, f"--zen-cache-host={ZEN_CACHE}", f"--zen-cache-upload={ZEN_CACHE_POPULATE}", f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}", - ]) + extra_zen_args=extra_zen_args, + )) print() print(f"--------- validating {build.name} oplog") - run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + run(zen_cmd("oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, + extra_zen_args=extra_zen_args)) print() - wipe_or_create("export folder", export_dir) + wipe_or_create("export folder", export_dir, extra_zen_args) - for build in BUILDS: + for build in builds: print(f"--------- exporting {build.name} oplog") - run([ - ZEN_EXE, "oplog-export", + run(zen_cmd( + "oplog-export", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, "--file", export_dir, "--forcetempblocks", - ]) + extra_zen_args=extra_zen_args, + )) print() finally: - stop_server("zenserver", ZEN_PORT) + stop_server("zenserver", ZEN_PORT, extra_zen_args) - wipe_or_create("alternate zenserver data", zen_import_data_dir) - start_server("import zenserver", zen_import_data_dir, ZEN_PORT) + wipe_or_create("alternate zenserver data", zen_import_data_dir, extra_zen_args) + start_server("import zenserver", zen_import_data_dir, ZEN_PORT, extra_zen_args=extra_zen_args) try: - setup_project(ZEN_PORT) + setup_project(ZEN_PORT, extra_zen_args) - for build in BUILDS: - setup_oplog(ZEN_PORT, build.name) + for build in builds: + setup_oplog(ZEN_PORT, build.name, extra_zen_args) print(f"--------- importing {build.name} oplog") - run([ - ZEN_EXE, "oplog-import", + run(zen_cmd( + "oplog-import", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, "--file", export_dir, - ]) + extra_zen_args=extra_zen_args, + )) print() print(f"--------- validating {build.name} oplog") - run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + run(zen_cmd("oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, + extra_zen_args=extra_zen_args)) print() finally: - stop_server("alternative zenserver", ZEN_PORT) + stop_server("alternative zenserver", ZEN_PORT, extra_zen_args) finally: - stop_server("cache zenserver", ZEN_CACHE_PORT) + stop_server("cache zenserver", ZEN_CACHE_PORT, extra_zen_args) if __name__ == "__main__": diff --git a/scripts/test_scripts/oplog-update-build-ids.py b/scripts/test_scripts/oplog-update-build-ids.py new file mode 100644 index 000000000..67e128c8e --- /dev/null +++ b/scripts/test_scripts/oplog-update-build-ids.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Update oplog-import-export-build-ids.json with build IDs at the highest common changelist across all buckets.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +import tempfile +from pathlib import Path + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" +_DEFAULT_ZEN = Path(f"build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") + + +def _cache_dir() -> Path: + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return base / "Temp" / "zen" + elif sys.platform == "darwin": + return Path.home() / "Library" / "Caches" / "zen" + else: + base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return base / "zen" + + +_OUTPUT_PATH = _cache_dir() / "oplog-import-export-build-ids.json" + +# Maps build name -> Jupiter bucket +_BUILDS: list[tuple[str, str]] = [ + ("XB1Client", "fortnitegame.oplog.fortnite-main.xb1client"), + ("WindowsClient", "fortnitegame.oplog.fortnite-main.windowsclient"), + ("SwitchClient", "fortnitegame.oplog.fortnite-main.switchclient"), + ("XSXClient", "fortnitegame.oplog.fortnite-main.xsxclient"), + ("Switch2Client", "fortnitegame.oplog.fortnite-main.switch2client"), + ("PS4Client", "fortnitegame.oplog.fortnite-main.ps4client"), + ("PS5Client", "fortnitegame.oplog.fortnite-main.ps5client"), + ("LinuxServer", "fortnitegame.oplog.fortnite-main.linuxserver"), + ("IOSClient", "fortnitegame.oplog.fortnite-main.iosclient"), + ("Android_ASTCClient", "fortnitegame.oplog.fortnite-main.android_astcclient"), +] + + +def list_builds_for_bucket(zen: str, host: str, namespace: str, bucket: str) -> list[dict]: + """Run zen builds list for a single bucket and return the results array.""" + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp: + result_path = Path(tmp.name) + + cmd = [ + zen, "builds", "list", + "--namespace", namespace, + "--bucket", bucket, + "--host", host, + "--result-path", str(result_path), + ] + + try: + subprocess.run(cmd, check=True, capture_output=True) + except FileNotFoundError: + sys.exit(f"error: zen binary not found: {zen}") + except subprocess.CalledProcessError as e: + sys.exit( + f"error: zen builds list failed for bucket '{bucket}' with exit code {e.returncode}\n" + f"stderr: {e.stderr.decode(errors='replace')}" + ) + + with result_path.open() as f: + data = json.load(f) + result_path.unlink(missing_ok=True) + + return data.get("results", []) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Refresh oplog-import-export-build-ids.json with build IDs at the highest changelist present in all buckets." + ) + parser.add_argument("--host", default="https://jupiter.devtools.epicgames.com", help="Jupiter host URL") + parser.add_argument("--zen", default=str(_DEFAULT_ZEN), help="Path to the zen binary") + parser.add_argument("--namespace", default="fortnite.oplog", help="Builds storage namespace") + args = parser.parse_args() + + # For each bucket, fetch results and build a changelist -> buildId map. + # bucket_cl_map[bucket] = { changelist_int: buildId_str, ... } + bucket_cl_map: dict[str, dict[int, str]] = {} + + for name, bucket in _BUILDS: + print(f"Querying {name} ({bucket}) ...") + results = list_builds_for_bucket(args.zen, args.host, args.namespace, bucket) + if not results: + sys.exit(f"error: no results for bucket '{bucket}' (build '{name}')") + + cl_map: dict[int, str] = {} + for entry in results: + build_id = entry.get("buildId", "") + metadata = entry.get("metadata") or {} + cl = metadata.get("changelist") + if build_id and cl is not None: + # Keep first occurrence (most recent) per changelist + if cl not in cl_map: + cl_map[int(cl)] = build_id + + if not cl_map: + sys.exit( + f"error: bucket '{bucket}' (build '{name}') returned {len(results)} entries " + "but none had both buildId and changelist in metadata" + ) + + print(f" {len(cl_map)} distinct changelists, latest CL {max(cl_map)}") + bucket_cl_map[bucket] = cl_map + + # Find the highest changelist present in every bucket's result set. + common_cls = set(next(iter(bucket_cl_map.values())).keys()) + for bucket, cl_map in bucket_cl_map.items(): + common_cls &= set(cl_map.keys()) + + if not common_cls: + sys.exit( + "error: no changelist is present in all buckets.\n" + "Per-bucket CL ranges:\n" + + "\n".join( + f" {name} ({bucket}): {min(bucket_cl_map[bucket])} – {max(bucket_cl_map[bucket])}" + for name, bucket in _BUILDS + ) + ) + + best_cl = max(common_cls) + print(f"\nHighest common changelist: {best_cl}") + + build_ids: dict[str, dict[str, str]] = {} + for name, bucket in _BUILDS: + build_id = bucket_cl_map[bucket][best_cl] + build_ids[name] = {"bucket": bucket, "buildId": build_id} + print(f" {name}: {build_id}") + + output = {"namespace": args.namespace, "builds": build_ids} + _OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + with _OUTPUT_PATH.open("w") as f: + json.dump(output, f, indent=2) + f.write("\n") + + print(f"\nWrote {_OUTPUT_PATH}") + + +if __name__ == "__main__": + main() -- cgit v1.2.3