From fb6bf42e0b074e9dd0a3e4f5f3e02782bf780f2c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 12 Feb 2026 15:29:13 +0100 Subject: bump sentry to 0.12.1 (#721) --- scripts/bundle.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/bundle.lua b/scripts/bundle.lua index debb1e615..07e120d04 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -176,7 +176,8 @@ local function main_windows(signidentity) "/v", "/as", "build/windows/x64/release/zenserver.exe", - "build/windows/x64/release/zen.exe") + "build/windows/x64/release/zen.exe", + "build/windows/x64/release/crashpad_handler.exe") if ret > 0 then raise("Failed signing zenserver binary") end -- cgit v1.2.3 From 794f093057c58c4909a0653edb54fdf869560596 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 14:00:34 +0100 Subject: native xmake toolchain definition for UE-clang (#805) This change is meant to provide a smoother experience when working on Linux. After this change, the toolchain setup process is now simply ```bash $ scripts/ue_build_linux/get_ue_toolchain.sh ``` and then at config time the toolchain is automatically detected if you downloaded it to the default location or have the `UE_TOOLCHAIN_DIR` environment variable set ```bash xmake config --mode=debug ``` Compared to the old script-based approach this configures the toolchain more precisely, avoiding leakage into unrelated build processes such as when a package manager decides to build something like Ninja locally etc. --- scripts/bundle.lua | 10 +++++++ scripts/ue_build_linux/README.md | 42 +++++++++++++++--------------- scripts/ue_build_linux/clang | 2 -- scripts/ue_build_linux/clang++ | 2 -- scripts/ue_build_linux/get_ue_toolchain.sh | 2 +- scripts/ue_build_linux/ue_build.sh | 33 ----------------------- 6 files changed, 32 insertions(+), 59 deletions(-) delete mode 100755 scripts/ue_build_linux/clang delete mode 100755 scripts/ue_build_linux/clang++ delete mode 100755 scripts/ue_build_linux/ue_build.sh (limited to 'scripts') diff --git a/scripts/bundle.lua b/scripts/bundle.lua index 07e120d04..b2a5e1e08 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -17,7 +17,15 @@ end -------------------------------------------------------------------------------- local function _build(arch, debug, config_args) + import("core.project.config") + config.load() + variant = debug and "debug" or "release" + + -- Preserve toolchain/sdk from current config so --clean doesn't lose them + local toolchain_arg = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or nil + local sdk_arg = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or nil + local ret = _exec( "xmake", "config", @@ -26,6 +34,8 @@ local function _build(arch, debug, config_args) "--mode="..variant, "--arch="..arch, "--zensentry=yes", + toolchain_arg, + sdk_arg, config_args) if ret > 0 then raise("Failed to configure xmake") diff --git a/scripts/ue_build_linux/README.md b/scripts/ue_build_linux/README.md index e93a234ae..afafcbe24 100755 --- a/scripts/ue_build_linux/README.md +++ b/scripts/ue_build_linux/README.md @@ -2,38 +2,38 @@ This folder contains scripts to build Zen using the UE Linux toolchain. This can be used to output binaries that meet the VFX Reference Platform versions. -It works by using the `--sysroot=` option to redirect compilers and linkers to -find headers and libraries. There are a few components involved; -1) get_ue_toolchain.sh +## Setup + +Download the toolchain using `get_ue_toolchain.sh`: ``` -$ scripts/ue_build_linux/get_ue_toolchain.sh ./.tmp-ue-toolchain +$ scripts/ue_build_linux/get_ue_toolchain.sh ``` -This will download the required components from cdn.unrealengine.com and -structure them in such a way that they can be used by both vcpkg and xmake -when building Zen. +By default this downloads to `~/.ue-toolchain`. A custom path can be given as +the first argument, or via the `UE_TOOLCHAIN_DIR` environment variable. -2) ue_build.sh [args...] +This will download the required components from cdn.unrealengine.com and +structure them in such a way that they can be used by xmake when building Zen. -Given the toolchain location downloaded in step (1) and the `VCPKG_ROOT` -environment variable is properly configured, this script sets up a suitable -environment and execs the "prog [args...]". +## Building -It is expected that this is used to invoke xmake to build Zen; +xmake automatically detects the toolchain at `~/.ue-toolchain`, so no extra +flags are needed: ``` -$ scripts/ue_build_linux/ue_build.sh .tmp-ue-toolchain xmake config --mode=debug -$ scripts/ue_build_linux/ue_build.sh .tmp-ue-toolchain xmake build +$ xmake config -y -m debug +$ xmake build -y ``` -It is possible that `--toolchain=clang` may be required as a configuration -option. The `ue_build.sh` script can also be sourced into the current shell, -although it is worth noting that this has never been tried. +To build a release bundle: -3) `scripts/ue_build_linux/clang` / `scripts/ue_build_linux/clang++` +``` +$ xmake config -y -m release +$ xmake bundle -y +``` -These acts as shims to the binaries in `toolchain_dir`, adding in the required -command line arguments to use the correct headers and libraries. -The `ue_build.sh` script adjusts `$PATH` appropriately. +The toolchain can also be selected explicitly with `--toolchain=ue-clang`, and +the SDK location can be overridden with `--sdk=` (must be absolute) or +the `UE_TOOLCHAIN_DIR` environment variable. diff --git a/scripts/ue_build_linux/clang b/scripts/ue_build_linux/clang deleted file mode 100755 index 9666ba4ba..000000000 --- a/scripts/ue_build_linux/clang +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec $UE_TOOLCHAIN_DIR/bin/clang --sysroot=$UE_TOOLCHAIN_DIR $CFLAGS "$@" diff --git a/scripts/ue_build_linux/clang++ b/scripts/ue_build_linux/clang++ deleted file mode 100755 index be106ae87..000000000 --- a/scripts/ue_build_linux/clang++ +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec $UE_TOOLCHAIN_DIR/bin/clang++ --sysroot=$UE_TOOLCHAIN_DIR -stdlib=libc++ $CXXFLAGS "$@" diff --git a/scripts/ue_build_linux/get_ue_toolchain.sh b/scripts/ue_build_linux/get_ue_toolchain.sh index c2538b09a..0afd40fe3 100755 --- a/scripts/ue_build_linux/get_ue_toolchain.sh +++ b/scripts/ue_build_linux/get_ue_toolchain.sh @@ -5,7 +5,7 @@ ZEN_ROOT=$(realpath $SCRIPT_DIR/../..) die() { echo "ERROR: $1"; exit; } -toolchain_dir="${1:-${ZEN_ROOT}/.tmp-ue-toolchain}" +toolchain_dir="${1:-${UE_TOOLCHAIN_DIR:-${HOME}/.ue-toolchain}}" if [[ $toolchain_dir == "--help" ]]; then echo "usage: $(basename ${BASH_SOURCE[0]}) " diff --git a/scripts/ue_build_linux/ue_build.sh b/scripts/ue_build_linux/ue_build.sh deleted file mode 100755 index 690f9f661..000000000 --- a/scripts/ue_build_linux/ue_build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -ZEN_ROOT=$(realpath $SCRIPT_DIR/../..) - -die() { echo ERROR: $1; exit 1; } - -if [[ $1 == "--help" ]]; then - echo "usage: $0 (defaults to ${ZEN_ROOT}/.tmp-ue-toolchain)" - exit -fi - -toolchain_dir="${1:-${ZEN_ROOT}/.tmp-ue-toolchain}" - -# Validate input - -if ! [ -d $toolchain_dir ]; then - die "$1 is not a directory" -fi - -if ! [ -e $toolchain_dir/bin/clang++ ]; then - die "$1/bin/clang++ does not exist" -fi - -export UE_TOOLCHAIN_DIR=$(realpath $toolchain_dir) -export CC="clang" -export CXX="clang++" -export LD="clang++" - -export PATH="$(realpath $(dirname ${BASH_SOURCE[0]})):$PATH" - -shift -exec "$@" -- cgit v1.2.3 From 2f0d60cb431ffefecf3e0a383528691be74af21b Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 5 Mar 2026 14:31:27 +0100 Subject: oidctoken tool package (#810) * added OidcToken binary to the build process. The binary is mirrored from p4 and is placed next to the output of the build process. It is also placed in the release zip archives. * also fixed issue with Linux symbol stripping which was introduced in toolchain changes yesterday --- scripts/bundle.lua | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/bundle.lua b/scripts/bundle.lua index b2a5e1e08..6f540c5b8 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -199,8 +199,9 @@ local function main_windows(signidentity) "build/windows/x64/release/zenserver.pdb", "build/windows/x64/release/zen.exe", "build/windows/x64/release/zen.pdb", - "build/windows/x64/release/crashpad_handler.exe") -end + "build/windows/x64/release/crashpad_handler.exe", + "build/windows/x64/release/oidctoken.exe") + end -------------------------------------------------------------------------------- local function main_mac(signidentity) @@ -277,7 +278,8 @@ local function main_mac(signidentity) "build/zenserver-macos.zip", "build/macosx/universal/release/zenserver", "build/macosx/universal/release/zen", - "build/macosx/universal/release/crashpad_handler") + "build/macosx/universal/release/crashpad_handler", + "build/macosx/x86_64/release/Oidctoken") end -------------------------------------------------------------------------------- @@ -288,7 +290,8 @@ local function main_linux() "build/zenserver-linux.zip", "build/linux/x86_64/release/zenserver", "build/linux/x86_64/release/zen", - "build/linux/x86_64/release/crashpad_handler") + "build/linux/x86_64/release/crashpad_handler", + "build/linux/x86_64/release/Oidctoken") end -------------------------------------------------------------------------------- -- cgit v1.2.3 From f9d8cbcb3573b47b639b7bd73d3a4eed17653d71 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Mar 2026 13:08:00 +0100 Subject: add fallback for zencache multirange (#816) * clean up BuildStorageResolveResult to allow capabilities * add check for multirange request capability * add MaxRangeCountPerRequest capabilities * project export tests * add InMemoryBuildStorageCache * progress and logging improvements * fix ElapsedSeconds calculations in fileremoteprojectstore.cpp * oplogs/builds test script --- .../test_scripts/builds-download-upload-test.py | 196 ++++++++++++++++++ scripts/test_scripts/metadatas/AndroidClient.json | 9 + scripts/test_scripts/metadatas/IOSClient.json | 9 + scripts/test_scripts/metadatas/LinuxServer.json | 9 + scripts/test_scripts/metadatas/PS4Client.json | 9 + scripts/test_scripts/metadatas/Switch2Client.json | 9 + scripts/test_scripts/metadatas/SwitchClient.json | 9 + scripts/test_scripts/metadatas/WindowsClient.json | 9 + scripts/test_scripts/metadatas/XB1Client.json | 9 + scripts/test_scripts/oplog-import-export-test.py | 228 +++++++++++++++++++++ 10 files changed, 496 insertions(+) create mode 100644 scripts/test_scripts/builds-download-upload-test.py create mode 100644 scripts/test_scripts/metadatas/AndroidClient.json create mode 100644 scripts/test_scripts/metadatas/IOSClient.json create mode 100644 scripts/test_scripts/metadatas/LinuxServer.json create mode 100644 scripts/test_scripts/metadatas/PS4Client.json create mode 100644 scripts/test_scripts/metadatas/Switch2Client.json create mode 100644 scripts/test_scripts/metadatas/SwitchClient.json create mode 100644 scripts/test_scripts/metadatas/WindowsClient.json create mode 100644 scripts/test_scripts/metadatas/XB1Client.json create mode 100644 scripts/test_scripts/oplog-import-export-test.py (limited to 'scripts') diff --git a/scripts/test_scripts/builds-download-upload-test.py b/scripts/test_scripts/builds-download-upload-test.py new file mode 100644 index 000000000..f03528d98 --- /dev/null +++ b/scripts/test_scripts/builds-download-upload-test.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Test script for builds download/upload operations.""" + +from __future__ import annotations + +import argparse +import platform +import subprocess +import sys +from pathlib import Path +from typing import NamedTuple + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" + + +class Build(NamedTuple): + name: str + bucket: str + id: str + + +BUILDS = [ + Build("XB1Client", "fortnitegame.staged-build.fortnite-main.xb1-client", "09a7616c1a388dfe6056aa57"), + Build("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client", "09a762c81e2cf213142d0ce5"), + Build("SwitchClient", "fortnitegame.staged-build.fortnite-main.switch-client", "09a75bf9c3ce75bce09f644f"), + Build("LinuxServer", "fortnitegame.staged-build.fortnite-main.linux-server", "09a750ac155eb3e3b62e87e0"), + Build("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client", "09a78f3df07b289691ec5710"), + Build("PS4Client", "fortnitegame.staged-build.fortnite-main.ps4-client", "09a76ea92ad301d4724fafad"), + Build("IOSClient", "fortnitegame.staged-build.fortnite-main.ios-client", "09a7816fa26c23362fef0c5d"), + Build("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client", "09a76725f1620d62c6be06e4"), +] + +ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") +ZEN_METADATA_DIR: Path = Path(__file__).resolve().parent / "metadatas" + +ZEN_PORT = 8558 +ZEN_CACHE_PORT = 8559 +ZEN_CACHE = f"http://127.0.0.1:{ZEN_CACHE_PORT}" +ZEN_PARTIAL_REQUEST_MODE = "true" + +SERVER_ARGS: tuple[str, ...] = ( + "--http", "asio", + "--gc-cache-duration-seconds", "1209600", + "--gc-interval-seconds", "21600", + "--gc-low-diskspace-threshold", "2147483648", + "--cache-bucket-limit-overwrites", +) + + +def run(cmd: list[str | Path]) -> None: + try: + subprocess.run(cmd, check=True) + except FileNotFoundError: + sys.exit(f"error: executable not found: {cmd[0]}") + except subprocess.CalledProcessError as e: + sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}") + + +def stop_server(label: str, port: int) -> None: + """Stop a zen server. Tolerates failures so it is safe to call from finally blocks.""" + print(f"--------- stopping {label}") + try: + subprocess.run([ZEN_EXE, "down", "--port", str(port)]) + except OSError as e: + print(f"warning: could not stop {label}: {e}", file=sys.stderr) + print() + + +def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None: + print(f"--------- starting {label} {data_dir}") + run([ + ZEN_EXE, "up", "--port", str(port), "--show-console", "--", + f"--data-dir={data_dir}", + *SERVER_ARGS, + *(extra_args or []), + ]) + print() + + +def wipe_or_create(label: str, path: Path) -> None: + if path.exists(): + print(f"--------- cleaning {label} {path}") + run([ZEN_EXE, "wipe", "-y", path]) + else: + print(f"--------- creating {label} {path}") + path.mkdir(parents=True, exist_ok=True) + print() + + +def check_prerequisites() -> None: + if not ZEN_EXE.is_file(): + sys.exit(f"error: zen executable not found: {ZEN_EXE}") + if not ZEN_METADATA_DIR.is_dir(): + sys.exit(f"error: metadata directory not found: {ZEN_METADATA_DIR}") + for build in BUILDS: + metadata = ZEN_METADATA_DIR / f"{build.name}.json" + if not metadata.is_file(): + sys.exit(f"error: metadata file not found: {metadata}") + + +def main() -> None: + global ZEN_EXE + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "positional_path", + nargs="?", + default=None, + type=Path, + metavar="DATA_PATH", + help="root path for all data directories (positional shorthand for --data-path)", + ) + parser.add_argument( + "zen_exe_positional", + nargs="?", + default=None, + type=Path, + metavar="ZEN_EXE_PATH", + help="path to zen executable (positional shorthand for --zen-exe-path)", + ) + parser.add_argument( + "--data-path", + default=Path(Path(__file__).stem + "_datadir"), + type=Path, + metavar="PATH", + help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + ) + parser.add_argument( + "--zen-exe-path", + default=ZEN_EXE, + type=Path, + metavar="PATH", + help=f"path to zen executable (default: {ZEN_EXE})", + ) + args = parser.parse_args() + + data_path = args.positional_path + if data_path is None: + data_path = args.data_path + + ZEN_EXE = args.zen_exe_positional + if ZEN_EXE is None: + ZEN_EXE = args.zen_exe_path + zen_system_dir = data_path / "system" + zen_download_dir = data_path / "Download" + zen_cache_data_dir = data_path / "ZenBuildsCache" + zen_upload_dir = data_path / "Upload" + zen_chunk_cache_path = data_path / "ChunkCache" + + check_prerequisites() + + start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"]) + try: + wipe_or_create("download folder", zen_download_dir) + wipe_or_create("system folder", zen_system_dir) + + for build in BUILDS: + print(f"--------- importing {build.name} build") + run([ + ZEN_EXE, "builds", "download", + "--host", "https://jupiter.devtools.epicgames.com", + "--namespace", "fortnite.oplog", + "--bucket", build.bucket, + "--build-id", build.id, + "--local-path", zen_download_dir / build.name, + f"--zen-cache-host={ZEN_CACHE}", + f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}", + "--verify", + "--system-dir", zen_system_dir, + ]) + print() + + wipe_or_create("upload folder", zen_upload_dir) + + for build in BUILDS: + print(f"--------- exporting {build.name} build") + run([ + ZEN_EXE, "builds", "upload", + "--storage-path", zen_upload_dir, + "--build-id", build.id, + "--local-path", zen_download_dir / build.name, + "--verify", + "--system-dir", zen_system_dir, + "--metadata-path", str(ZEN_METADATA_DIR / f"{build.name}.json"), + "--create-build", + "--chunking-cache-path", zen_chunk_cache_path, + ]) + print() + finally: + stop_server("cache zenserver", ZEN_CACHE_PORT) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_scripts/metadatas/AndroidClient.json b/scripts/test_scripts/metadatas/AndroidClient.json new file mode 100644 index 000000000..378d0454d --- /dev/null +++ b/scripts/test_scripts/metadatas/AndroidClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 AndroidClient", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Android", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/IOSClient.json b/scripts/test_scripts/metadatas/IOSClient.json new file mode 100644 index 000000000..fb0f9a342 --- /dev/null +++ b/scripts/test_scripts/metadatas/IOSClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 IOSClient", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "IOS", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/LinuxServer.json b/scripts/test_scripts/metadatas/LinuxServer.json new file mode 100644 index 000000000..02ae2d970 --- /dev/null +++ b/scripts/test_scripts/metadatas/LinuxServer.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 LinuxServer", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Linux", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/PS4Client.json b/scripts/test_scripts/metadatas/PS4Client.json new file mode 100644 index 000000000..6e49e3e5e --- /dev/null +++ b/scripts/test_scripts/metadatas/PS4Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 PS4Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "PS4", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/Switch2Client.json b/scripts/test_scripts/metadatas/Switch2Client.json new file mode 100644 index 000000000..41732e7bc --- /dev/null +++ b/scripts/test_scripts/metadatas/Switch2Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 Switch2Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Switch2", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/SwitchClient.json b/scripts/test_scripts/metadatas/SwitchClient.json new file mode 100644 index 000000000..49362f23e --- /dev/null +++ b/scripts/test_scripts/metadatas/SwitchClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 SwitchClient", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Switch", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/WindowsClient.json b/scripts/test_scripts/metadatas/WindowsClient.json new file mode 100644 index 000000000..c7af270c2 --- /dev/null +++ b/scripts/test_scripts/metadatas/WindowsClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 Windows Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Windows", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/XB1Client.json b/scripts/test_scripts/metadatas/XB1Client.json new file mode 100644 index 000000000..36fb45801 --- /dev/null +++ b/scripts/test_scripts/metadatas/XB1Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 XB1Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "XB1", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/oplog-import-export-test.py b/scripts/test_scripts/oplog-import-export-test.py new file mode 100644 index 000000000..51593d5a9 --- /dev/null +++ b/scripts/test_scripts/oplog-import-export-test.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Test script for oplog import/export operations.""" + +from __future__ import annotations + +import argparse +import platform +import subprocess +import sys +from pathlib import Path +from typing import NamedTuple + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" + + +class Build(NamedTuple): + name: str + bucket: str + id: str + + +BUILDS = [ + Build("XB1Client", "fortnitegame.oplog.fortnite-main.xb1client", "09a75f7f3b7517653dcdaaa4"), + Build("WindowsClient", "fortnitegame.oplog.fortnite-main.windowsclient", "09a75d977ef944ecfd0eddfd"), + Build("SwitchClient", "fortnitegame.oplog.fortnite-main.switchclient", "09a74d03b3598ec94cfd2644"), + Build("XSXClient", "fortnitegame.oplog.fortnite-main.xsxclient", "09a76c2bbd6cd78f4d40d9ea"), + Build("Switch2Client", "fortnitegame.oplog.fortnite-main.switch2client", "09a7686b3d9faa78fb24a38f"), + Build("PS4Client", "fortnitegame.oplog.fortnite-main.ps4client", "09a75b72d1c260ed26020140"), + Build("LinuxServer", "fortnitegame.oplog.fortnite-main.linuxserver", "09a747f5e0ee83a04be013e6"), + Build("IOSClient", "fortnitegame.oplog.fortnite-main.iosclient", "09a75f677e883325a209148c"), + Build("Android_ASTCClient", "fortnitegame.oplog.fortnite-main.android_astcclient", "09a7422c08c6f37becc7d37f"), +] + +ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") + +ZEN_PORT = 8558 +ZEN_CACHE_PORT = 8559 +ZEN_CACHE = f"http://127.0.0.1:{ZEN_CACHE_PORT}" +ZEN_CACHE_POPULATE = "true" +ZEN_PARTIAL_REQUEST_MODE = "true" + +SERVER_ARGS: tuple[str, ...] = ( + "--http", "asio", + "--gc-cache-duration-seconds", "1209600", + "--gc-interval-seconds", "21600", + "--gc-low-diskspace-threshold", "2147483648", + "--cache-bucket-limit-overwrites", +) + + +def run(cmd: list[str | Path]) -> None: + try: + subprocess.run(cmd, check=True) + except FileNotFoundError: + sys.exit(f"error: executable not found: {cmd[0]}") + except subprocess.CalledProcessError as e: + sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}") + + +def stop_server(label: str, port: int) -> None: + """Stop a zen server. Tolerates failures so it is safe to call from finally blocks.""" + print(f"--------- stopping {label}") + try: + subprocess.run([ZEN_EXE, "down", "--port", str(port)]) + except OSError as e: + print(f"warning: could not stop {label}: {e}", file=sys.stderr) + print() + + +def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None: + print(f"--------- starting {label} {data_dir}") + run([ + ZEN_EXE, "up", "--port", str(port), "--show-console", "--", + f"--data-dir={data_dir}", + *SERVER_ARGS, + *(extra_args or []), + ]) + print() + + +def wipe_or_create(label: str, path: Path) -> None: + if path.exists(): + print(f"--------- cleaning {label} {path}") + run([ZEN_EXE, "wipe", "-y", path]) + else: + print(f"--------- creating {label} {path}") + path.mkdir(parents=True, exist_ok=True) + print() + + +def check_prerequisites() -> None: + if not ZEN_EXE.is_file(): + sys.exit(f"error: zen executable not found: {ZEN_EXE}") + + +def setup_project(port: int) -> None: + """Create the FortniteGame project and all oplogs on the server at the given port.""" + print("--------- creating FortniteGame project") + run([ZEN_EXE, "project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update"]) + print() + + for build in BUILDS: + print(f"--------- creating {build.name} oplog") + run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build.name, "--force-update"]) + print() + + +def main() -> None: + global ZEN_EXE + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "positional_path", + nargs="?", + default=None, + type=Path, + metavar="DATA_PATH", + help="root path for all data directories (positional shorthand for --data-path)", + ) + parser.add_argument( + "zen_exe_positional", + nargs="?", + default=None, + type=Path, + metavar="ZEN_EXE_PATH", + help="path to zen executable (positional shorthand for --zen-exe-path)", + ) + parser.add_argument( + "--data-path", + default=Path(Path(__file__).stem + "_datadir"), + type=Path, + metavar="PATH", + help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + ) + parser.add_argument( + "--zen-exe-path", + default=ZEN_EXE, + type=Path, + metavar="PATH", + help=f"path to zen executable (default: {ZEN_EXE})", + ) + args = parser.parse_args() + + data_path = args.positional_path + if data_path is None: + data_path = args.data_path + + ZEN_EXE = args.zen_exe_positional + if ZEN_EXE is None: + ZEN_EXE = args.zen_exe_path + zen_data_dir = data_path / "DDC" / "OplogsZen" + zen_cache_data_dir = data_path / "DDC" / "ZenBuildsCache" + zen_import_data_dir = data_path / "DDC" / "OplogsZenImport" + export_dir = data_path / "Export" / "FortniteGame" + + check_prerequisites() + + start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"]) + try: + wipe_or_create("zenserver data", zen_data_dir) + start_server("zenserver", zen_data_dir, ZEN_PORT) + try: + setup_project(ZEN_PORT) + + for build in BUILDS: + print(f"--------- importing {build.name} oplog") + run([ + ZEN_EXE, "oplog-import", + f"--hosturl=127.0.0.1:{ZEN_PORT}", + "FortniteGame", build.name, + "--clean", + "--builds", "https://jupiter.devtools.epicgames.com", + "--namespace", "fortnite.oplog", + "--bucket", build.bucket, + "--builds-id", build.id, + f"--zen-cache-host={ZEN_CACHE}", + f"--zen-cache-upload={ZEN_CACHE_POPULATE}", + f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}", + ]) + print() + + print(f"--------- validating {build.name} oplog") + run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + print() + + wipe_or_create("export folder", export_dir) + + for build in BUILDS: + print(f"--------- exporting {build.name} oplog") + run([ + ZEN_EXE, "oplog-export", + f"--hosturl=127.0.0.1:{ZEN_PORT}", + "FortniteGame", build.name, + "--file", export_dir, + "--forcetempblocks", + ]) + print() + finally: + stop_server("zenserver", ZEN_PORT) + + wipe_or_create("alternate zenserver data", zen_import_data_dir) + start_server("import zenserver", zen_import_data_dir, ZEN_PORT) + try: + setup_project(ZEN_PORT) + + for build in BUILDS: + print(f"--------- importing {build.name} oplog") + run([ + ZEN_EXE, "oplog-import", + f"--hosturl=127.0.0.1:{ZEN_PORT}", + "FortniteGame", build.name, + "--file", export_dir, + ]) + print() + + print(f"--------- validating {build.name} oplog") + run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + print() + finally: + stop_server("alternative zenserver", ZEN_PORT) + finally: + stop_server("cache zenserver", ZEN_CACHE_PORT) + + +if __name__ == "__main__": + main() -- cgit v1.2.3 From 9f33eaa7b7f246d68ea539f32cd74ecf8dfd5790 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Mar 2026 22:20:52 +0100 Subject: =?UTF-8?q?updated=20chunk=E2=80=93block=20analyser=20(#818)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * create oplogs as they are imported * Improved logic for partial block analisys * unit tests for ChunkBlockAnalyser --- scripts/test_scripts/oplog-import-export-test.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/test_scripts/oplog-import-export-test.py b/scripts/test_scripts/oplog-import-export-test.py index 51593d5a9..b2a5ece6c 100644 --- a/scripts/test_scripts/oplog-import-export-test.py +++ b/scripts/test_scripts/oplog-import-export-test.py @@ -96,15 +96,17 @@ def check_prerequisites() -> None: def setup_project(port: int) -> None: - """Create the FortniteGame project and all oplogs on the server at the given port.""" + """Create the FortniteGame project on the server at the given port.""" print("--------- creating FortniteGame project") run([ZEN_EXE, "project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update"]) print() - for build in BUILDS: - print(f"--------- creating {build.name} oplog") - run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build.name, "--force-update"]) - print() + +def setup_oplog(port: int, build_name: str) -> None: + """Create the oplog in the FortniteGame project on the server at the given port.""" + print(f"--------- creating {build_name} oplog") + run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build_name, "--force-update"]) + print() def main() -> None: @@ -165,6 +167,8 @@ def main() -> None: setup_project(ZEN_PORT) for build in BUILDS: + setup_oplog(ZEN_PORT, build.name) + print(f"--------- importing {build.name} oplog") run([ ZEN_EXE, "oplog-import", @@ -206,6 +210,8 @@ def main() -> None: setup_project(ZEN_PORT) for build in BUILDS: + setup_oplog(ZEN_PORT, build.name) + print(f"--------- importing {build.name} oplog") run([ ZEN_EXE, "oplog-import", -- cgit v1.2.3