diff options
| author | Stefan Boberg <[email protected]> | 2026-03-15 22:53:37 +0100 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2026-03-15 22:53:37 +0100 |
| commit | da216166aab918bc93ee73040c1eb6336ea6d026 (patch) | |
| tree | 87263e1e70d975c640981a50474290cdad1382cc | |
| parent | Change --noclean to --clean in toolchain verify script (diff) | |
| parent | Merge branch 'main' into sb/linux-build-improvements (diff) | |
| download | zen-sb/linux-build-improvements.tar.xz zen-sb/linux-build-improvements.zip | |
Merge branch 'sb/linux-build-improvements' of ssh://arn-wd-l1704.localdomain:2222/ue-foundation/zen into sb/linux-build-improvementssb/linux-build-improvements
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | CLAUDE.md | 4 | ||||
| -rw-r--r-- | scripts/test_scripts/builds-download-upload-test.py | 74 | ||||
| -rw-r--r-- | scripts/test_scripts/builds-download-upload-update-build-ids.py | 150 | ||||
| -rw-r--r-- | scripts/test_scripts/oplog-import-export-test.py | 177 | ||||
| -rw-r--r-- | scripts/test_scripts/oplog-update-build-ids.py | 151 | ||||
| -rw-r--r-- | scripts/updatefrontend.lua | 111 | ||||
| -rw-r--r-- | src/transports/winsock/winsock.cpp | 4 | ||||
| -rw-r--r-- | src/zencore/include/zencore/logbase.h | 2 | ||||
| -rw-r--r-- | src/zencore/include/zencore/string.h | 2 | ||||
| -rw-r--r-- | src/zencore/memtrack/callstacktrace.cpp | 146 | ||||
| -rw-r--r-- | src/zenhttp/httpclient.cpp | 19 | ||||
| -rw-r--r-- | src/zenhttp/include/zenhttp/cprutils.h | 22 | ||||
| -rw-r--r-- | src/zenhttp/include/zenhttp/httpclient.h | 2 | ||||
| -rw-r--r-- | src/zenhttp/servers/httpplugin.cpp | 2 | ||||
| -rw-r--r-- | src/zenhttp/xmake.lua | 7 | ||||
| -rw-r--r-- | src/zenserver/frontend/html.zip | bin | 431365 -> 0 bytes | |||
| -rw-r--r-- | src/zenserver/xmake.lua | 48 | ||||
| -rw-r--r-- | src/zenstore/xmake.lua | 7 | ||||
| -rw-r--r-- | src/zenutil/workerpools.cpp | 2 | ||||
| -rw-r--r-- | thirdparty/xmake.lua | 4 | ||||
| -rw-r--r-- | xmake.lua | 45 |
22 files changed, 685 insertions, 297 deletions
diff --git a/.gitignore b/.gitignore index 1073d5513..5c9195566 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,6 @@ CMake* # Ue tool chain temp directory .tmp-ue-toolchain/ + +# Generated frontend zip (built automatically by xmake) +src/zenserver/frontend/html.zip @@ -173,7 +173,7 @@ The codebase is organized into layered modules with clear dependencies: - Web UI bundled as ZIP in `src/zenserver/frontend/*.zip` - Dashboards for hub, orchestrator, and compute services are located in `src/zenserver/frontent/html/` - These are the files which end up being bundled into the front-end zip mentioned above -- Update with `xmake updatefrontend` after modifying HTML/JS, and check in the resulting zip +- The zip is generated automatically at build time when source files change **Memory Management:** - Can use mimalloc or rpmalloc for performance @@ -309,6 +309,4 @@ When debugging zenserver-test or other multi-process scenarios, use child proces # Create deployable ZIP bundle xmake bundle -# Update frontend ZIP after HTML changes -xmake updatefrontend ``` diff --git a/scripts/test_scripts/builds-download-upload-test.py b/scripts/test_scripts/builds-download-upload-test.py index e4fee7cb8..8ff5245c1 100644 --- a/scripts/test_scripts/builds-download-upload-test.py +++ b/scripts/test_scripts/builds-download-upload-test.py @@ -4,6 +4,8 @@ from __future__ import annotations import argparse +import json +import os import platform import subprocess import sys @@ -15,22 +17,51 @@ _ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() _EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" +def _cache_dir() -> Path: + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return base / "Temp" / "zen" + elif sys.platform == "darwin": + return Path.home() / "Library" / "Caches" / "zen" + else: + base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return base / "zen" + + +_BUILD_IDS_PATH = _cache_dir() / "builds-download-upload-build-ids.json" + + class Build(NamedTuple): name: str bucket: str id: str -BUILDS = [ - Build("XB1Client", "fortnitegame.staged-build.fortnite-main.xb1-client", "09a7616c1a388dfe6056aa57"), - Build("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client", "09a762c81e2cf213142d0ce5"), - Build("SwitchClient", "fortnitegame.staged-build.fortnite-main.switch-client", "09a75bf9c3ce75bce09f644f"), - Build("LinuxServer", "fortnitegame.staged-build.fortnite-main.linux-server", "09a750ac155eb3e3b62e87e0"), - Build("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client", "09a78f3df07b289691ec5710"), - Build("PS4Client", "fortnitegame.staged-build.fortnite-main.ps4-client", "09a76ea92ad301d4724fafad"), - Build("IOSClient", "fortnitegame.staged-build.fortnite-main.ios-client", "09a7816fa26c23362fef0c5d"), - Build("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client", "09a76725f1620d62c6be06e4"), -] +def load_builds() -> tuple[str, list[Build]]: + if not _BUILD_IDS_PATH.exists(): + print(f"Build IDs file not found: {_BUILD_IDS_PATH}") + answer = input("Run builds-download-upload-update-build-ids.py now to populate it? [y/N] ").strip().lower() + if answer == "y": + update_script = Path(__file__).parent / "builds-download-upload-update-build-ids.py" + subprocess.run([sys.executable, str(update_script)], check=True) + else: + sys.exit("Aborted. Run scripts/test_scripts/builds-download-upload-update-build-ids.py to populate it.") + with _BUILD_IDS_PATH.open() as f: + data: dict = json.load(f) + namespace = data.get("namespace", "") + if not namespace: + sys.exit(f"error: {_BUILD_IDS_PATH} is missing 'namespace'") + builds = [] + for name, entry in data.get("builds", {}).items(): + bucket = entry.get("bucket", "") + build_id = entry.get("buildId", "") + if not bucket or not build_id: + sys.exit(f"error: entry '{name}' in {_BUILD_IDS_PATH} is missing 'bucket' or 'buildId'") + builds.append(Build(name, bucket, build_id)) + if not builds: + sys.exit(f"error: {_BUILD_IDS_PATH} contains no builds") + return namespace, builds + ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") ZEN_METADATA_DIR: Path = Path(__file__).resolve().parent / "metadatas" @@ -99,12 +130,12 @@ def wipe_or_create(label: str, path: Path, extra_zen_args: list[str] | None = No print() -def check_prerequisites() -> None: +def check_prerequisites(builds: list[Build]) -> None: if not ZEN_EXE.is_file(): sys.exit(f"error: zen executable not found: {ZEN_EXE}") if not ZEN_METADATA_DIR.is_dir(): sys.exit(f"error: metadata directory not found: {ZEN_METADATA_DIR}") - for build in BUILDS: + for build in builds: metadata = ZEN_METADATA_DIR / f"{build.name}.json" if not metadata.is_file(): sys.exit(f"error: metadata file not found: {metadata}") @@ -145,10 +176,10 @@ def main() -> None: ) parser.add_argument( "--data-path", - default=Path(Path(__file__).stem + "_datadir"), + default=None, type=Path, metavar="PATH", - help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + help="root path for all data directories", ) parser.add_argument( "--zen-exe-path", @@ -162,17 +193,24 @@ def main() -> None: data_path = args.positional_path if data_path is None: data_path = args.data_path + if data_path is None: + print("WARNING: This script may require up to 1TB of free disk space.") + raw = input("Enter root path for all data directories: ").strip() + if not raw: + sys.exit("error: data path is required") + data_path = Path(raw) ZEN_EXE = args.zen_exe_positional if ZEN_EXE is None: ZEN_EXE = args.zen_exe_path + namespace, builds = load_builds() zen_system_dir = data_path / "system" zen_download_dir = data_path / "Download" zen_cache_data_dir = data_path / "ZenBuildsCache" zen_upload_dir = data_path / "Upload" zen_chunk_cache_path = data_path / "ChunkCache" - check_prerequisites() + check_prerequisites(builds) start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, extra_zen_args=extra_zen_args, extra_server_args=["--buildstore-enabled"]) @@ -180,12 +218,12 @@ def main() -> None: wipe_or_create("download folder", zen_download_dir, extra_zen_args) wipe_or_create("system folder", zen_system_dir, extra_zen_args) - for build in BUILDS: + for build in builds: print(f"--------- importing {build.name} build") run(zen_cmd( "builds", "download", "--host", "https://jupiter.devtools.epicgames.com", - "--namespace", "fortnite.oplog", + "--namespace", namespace, "--bucket", build.bucket, "--build-id", build.id, "--local-path", zen_download_dir / build.name, @@ -199,7 +237,7 @@ def main() -> None: wipe_or_create("upload folder", zen_upload_dir, extra_zen_args) - for build in BUILDS: + for build in builds: print(f"--------- exporting {build.name} build") run(zen_cmd( "builds", "upload", diff --git a/scripts/test_scripts/builds-download-upload-update-build-ids.py b/scripts/test_scripts/builds-download-upload-update-build-ids.py new file mode 100644 index 000000000..2a63aa44d --- /dev/null +++ b/scripts/test_scripts/builds-download-upload-update-build-ids.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Update builds-download-upload-build-ids.json with build IDs at the highest common changelist across all buckets.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +import tempfile +from pathlib import Path + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" +_DEFAULT_ZEN = Path(f"build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") + + +def _cache_dir() -> Path: + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return base / "Temp" / "zen" + elif sys.platform == "darwin": + return Path.home() / "Library" / "Caches" / "zen" + else: + base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return base / "zen" + + +_OUTPUT_PATH = _cache_dir() / "builds-download-upload-build-ids.json" + +# Maps build name -> Jupiter bucket +_BUILDS: list[tuple[str, str]] = [ + ("XB1Client", "fortnitegame.staged-build.fortnite-main.xb1-client"), + ("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client"), + ("SwitchClient", "fortnitegame.staged-build.fortnite-main.switch-client"), + ("LinuxServer", "fortnitegame.staged-build.fortnite-main.linux-server"), + ("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client"), + ("PS4Client", "fortnitegame.staged-build.fortnite-main.ps4-client"), + ("PS5Client", "fortnitegame.staged-build.fortnite-main.ps5-client"), + ("IOSClient", "fortnitegame.staged-build.fortnite-main.ios-client"), + ("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client"), +] + + +def list_builds_for_bucket(zen: str, host: str, namespace: str, bucket: str) -> list[dict]: + """Run zen builds list for a single bucket and return the results array.""" + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp: + result_path = Path(tmp.name) + + cmd = [ + zen, "builds", "list", + "--namespace", namespace, + "--bucket", bucket, + "--host", host, + "--result-path", str(result_path), + ] + + try: + subprocess.run(cmd, check=True, capture_output=True) + except FileNotFoundError: + sys.exit(f"error: zen binary not found: {zen}") + except subprocess.CalledProcessError as e: + sys.exit( + f"error: zen builds list failed for bucket '{bucket}' with exit code {e.returncode}\n" + f"stderr: {e.stderr.decode(errors='replace')}" + ) + + with result_path.open() as f: + data = json.load(f) + result_path.unlink(missing_ok=True) + + return data.get("results", []) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Refresh builds-download-upload-build-ids.json with build IDs at the highest changelist present in all buckets." + ) + parser.add_argument("--host", default="https://jupiter.devtools.epicgames.com", help="Jupiter host URL") + parser.add_argument("--zen", default=str(_DEFAULT_ZEN), help="Path to the zen binary") + parser.add_argument("--namespace", default="fortnite.oplog", help="Builds storage namespace") + args = parser.parse_args() + + # For each bucket, fetch results and build a changelist -> buildId map. + # bucket_cl_map[bucket] = { changelist_int: buildId_str, ... } + bucket_cl_map: dict[str, dict[int, str]] = {} + + for name, bucket in _BUILDS: + print(f"Querying {name} ({bucket}) ...") + results = list_builds_for_bucket(args.zen, args.host, args.namespace, bucket) + if not results: + sys.exit(f"error: no results for bucket '{bucket}' (build '{name}')") + + cl_map: dict[int, str] = {} + for entry in results: + build_id = entry.get("buildId", "") + metadata = entry.get("metadata") or {} + cl = metadata.get("commit") + if build_id and cl is not None: + # Keep first occurrence (most recent) per changelist + if cl not in cl_map: + cl_map[int(cl)] = build_id + + if not cl_map: + sys.exit( + f"error: bucket '{bucket}' (build '{name}') returned {len(results)} entries " + "but none had both buildId and changelist in metadata" + ) + + print(f" {len(cl_map)} distinct changelists, latest CL {max(cl_map)}") + bucket_cl_map[bucket] = cl_map + + # Find the highest changelist present in every bucket's result set. + common_cls = set(next(iter(bucket_cl_map.values())).keys()) + for bucket, cl_map in bucket_cl_map.items(): + common_cls &= set(cl_map.keys()) + + if not common_cls: + sys.exit( + "error: no changelist is present in all buckets.\n" + "Per-bucket CL ranges:\n" + + "\n".join( + f" {name} ({bucket}): {min(bucket_cl_map[bucket])} – {max(bucket_cl_map[bucket])}" + for name, bucket in _BUILDS + ) + ) + + best_cl = max(common_cls) + print(f"\nHighest common changelist: {best_cl}") + + build_ids: dict[str, dict[str, str]] = {} + for name, bucket in _BUILDS: + build_id = bucket_cl_map[bucket][best_cl] + build_ids[name] = {"bucket": bucket, "buildId": build_id} + print(f" {name}: {build_id}") + + output = {"namespace": args.namespace, "builds": build_ids} + _OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + with _OUTPUT_PATH.open("w") as f: + json.dump(output, f, indent=2) + f.write("\n") + + print(f"\nWrote {_OUTPUT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_scripts/oplog-import-export-test.py b/scripts/test_scripts/oplog-import-export-test.py index b2a5ece6c..f913a7351 100644 --- a/scripts/test_scripts/oplog-import-export-test.py +++ b/scripts/test_scripts/oplog-import-export-test.py @@ -4,6 +4,8 @@ from __future__ import annotations import argparse +import json +import os import platform import subprocess import sys @@ -15,23 +17,51 @@ _ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() _EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" +def _cache_dir() -> Path: + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return base / "Temp" / "zen" + elif sys.platform == "darwin": + return Path.home() / "Library" / "Caches" / "zen" + else: + base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return base / "zen" + + +_BUILD_IDS_PATH = _cache_dir() / "oplog-import-export-build-ids.json" + + class Build(NamedTuple): name: str bucket: str id: str -BUILDS = [ - Build("XB1Client", "fortnitegame.oplog.fortnite-main.xb1client", "09a75f7f3b7517653dcdaaa4"), - Build("WindowsClient", "fortnitegame.oplog.fortnite-main.windowsclient", "09a75d977ef944ecfd0eddfd"), - Build("SwitchClient", "fortnitegame.oplog.fortnite-main.switchclient", "09a74d03b3598ec94cfd2644"), - Build("XSXClient", "fortnitegame.oplog.fortnite-main.xsxclient", "09a76c2bbd6cd78f4d40d9ea"), - Build("Switch2Client", "fortnitegame.oplog.fortnite-main.switch2client", "09a7686b3d9faa78fb24a38f"), - Build("PS4Client", "fortnitegame.oplog.fortnite-main.ps4client", "09a75b72d1c260ed26020140"), - Build("LinuxServer", "fortnitegame.oplog.fortnite-main.linuxserver", "09a747f5e0ee83a04be013e6"), - Build("IOSClient", "fortnitegame.oplog.fortnite-main.iosclient", "09a75f677e883325a209148c"), - Build("Android_ASTCClient", "fortnitegame.oplog.fortnite-main.android_astcclient", "09a7422c08c6f37becc7d37f"), -] +def load_builds() -> tuple[str, list[Build]]: + if not _BUILD_IDS_PATH.exists(): + print(f"Build IDs file not found: {_BUILD_IDS_PATH}") + answer = input("Run oplog-update-build-ids.py now to populate it? [y/N] ").strip().lower() + if answer == "y": + update_script = Path(__file__).parent / "oplog-update-build-ids.py" + subprocess.run([sys.executable, str(update_script)], check=True) + else: + sys.exit("Aborted. Run scripts/test_scripts/oplog-update-build-ids.py to populate it.") + with _BUILD_IDS_PATH.open() as f: + data: dict = json.load(f) + namespace = data.get("namespace", "") + if not namespace: + sys.exit(f"error: {_BUILD_IDS_PATH} is missing 'namespace'") + builds = [] + for name, entry in data.get("builds", {}).items(): + bucket = entry.get("bucket", "") + build_id = entry.get("buildId", "") + if not bucket or not build_id: + sys.exit(f"error: entry '{name}' in {_BUILD_IDS_PATH} is missing 'bucket' or 'buildId'") + builds.append(Build(name, bucket, build_id)) + if not builds: + sys.exit(f"error: {_BUILD_IDS_PATH} contains no builds") + return namespace, builds + ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") @@ -50,6 +80,11 @@ SERVER_ARGS: tuple[str, ...] = ( ) +def zen_cmd(*args: str | Path, extra_zen_args: list[str] | None = None) -> list[str | Path]: + """Build a zen CLI command list, inserting extra_zen_args before subcommands.""" + return [ZEN_EXE, *(extra_zen_args or []), *args] + + def run(cmd: list[str | Path]) -> None: try: subprocess.run(cmd, check=True) @@ -59,31 +94,33 @@ def run(cmd: list[str | Path]) -> None: sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}") -def stop_server(label: str, port: int) -> None: +def stop_server(label: str, port: int, extra_zen_args: list[str] | None = None) -> None: """Stop a zen server. Tolerates failures so it is safe to call from finally blocks.""" print(f"--------- stopping {label}") try: - subprocess.run([ZEN_EXE, "down", "--port", str(port)]) + subprocess.run(zen_cmd("down", "--port", str(port), extra_zen_args=extra_zen_args)) except OSError as e: print(f"warning: could not stop {label}: {e}", file=sys.stderr) print() -def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None: +def start_server(label: str, data_dir: Path, port: int, extra_zen_args: list[str] | None = None, + extra_server_args: list[str] | None = None) -> None: print(f"--------- starting {label} {data_dir}") - run([ - ZEN_EXE, "up", "--port", str(port), "--show-console", "--", + run(zen_cmd( + "up", "--port", str(port), "--show-console", "--", f"--data-dir={data_dir}", *SERVER_ARGS, - *(extra_args or []), - ]) + *(extra_server_args or []), + extra_zen_args=extra_zen_args, + )) print() -def wipe_or_create(label: str, path: Path) -> None: +def wipe_or_create(label: str, path: Path, extra_zen_args: list[str] | None = None) -> None: if path.exists(): print(f"--------- cleaning {label} {path}") - run([ZEN_EXE, "wipe", "-y", path]) + run(zen_cmd("wipe", "-y", path, extra_zen_args=extra_zen_args)) else: print(f"--------- creating {label} {path}") path.mkdir(parents=True, exist_ok=True) @@ -95,24 +132,39 @@ def check_prerequisites() -> None: sys.exit(f"error: zen executable not found: {ZEN_EXE}") -def setup_project(port: int) -> None: +def setup_project(port: int, extra_zen_args: list[str] | None = None) -> None: """Create the FortniteGame project on the server at the given port.""" print("--------- creating FortniteGame project") - run([ZEN_EXE, "project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update"]) + run(zen_cmd("project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update", + extra_zen_args=extra_zen_args)) print() -def setup_oplog(port: int, build_name: str) -> None: +def setup_oplog(port: int, build_name: str, extra_zen_args: list[str] | None = None) -> None: """Create the oplog in the FortniteGame project on the server at the given port.""" print(f"--------- creating {build_name} oplog") - run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build_name, "--force-update"]) + run(zen_cmd("oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build_name, "--force-update", + extra_zen_args=extra_zen_args)) print() def main() -> None: global ZEN_EXE - parser = argparse.ArgumentParser(description=__doc__) + # Split on '--' to separate script args from extra zen CLI args + script_argv: list[str] = [] + extra_zen_args: list[str] = [] + if "--" in sys.argv[1:]: + sep = sys.argv.index("--", 1) + script_argv = sys.argv[1:sep] + extra_zen_args = sys.argv[sep + 1:] + else: + script_argv = sys.argv[1:] + + parser = argparse.ArgumentParser( + description=__doc__, + epilog="Any arguments after '--' are forwarded to every zen CLI invocation.", + ) parser.add_argument( "positional_path", nargs="?", @@ -131,10 +183,10 @@ def main() -> None: ) parser.add_argument( "--data-path", - default=Path(Path(__file__).stem + "_datadir"), + default=None, type=Path, metavar="PATH", - help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + help="root path for all data directories", ) parser.add_argument( "--zen-exe-path", @@ -143,15 +195,22 @@ def main() -> None: metavar="PATH", help=f"path to zen executable (default: {ZEN_EXE})", ) - args = parser.parse_args() + args = parser.parse_args(script_argv) data_path = args.positional_path if data_path is None: data_path = args.data_path + if data_path is None: + print("WARNING: This script may require up to 1TB of free disk space.") + raw = input("Enter root path for all data directories: ").strip() + if not raw: + sys.exit("error: data path is required") + data_path = Path(raw) ZEN_EXE = args.zen_exe_positional if ZEN_EXE is None: ZEN_EXE = args.zen_exe_path + namespace, builds = load_builds() zen_data_dir = data_path / "DDC" / "OplogsZen" zen_cache_data_dir = data_path / "DDC" / "ZenBuildsCache" zen_import_data_dir = data_path / "DDC" / "OplogsZenImport" @@ -159,75 +218,81 @@ def main() -> None: check_prerequisites() - start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"]) + start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, + extra_zen_args=extra_zen_args, extra_server_args=["--buildstore-enabled"]) try: - wipe_or_create("zenserver data", zen_data_dir) - start_server("zenserver", zen_data_dir, ZEN_PORT) + wipe_or_create("zenserver data", zen_data_dir, extra_zen_args) + start_server("zenserver", zen_data_dir, ZEN_PORT, extra_zen_args=extra_zen_args) try: - setup_project(ZEN_PORT) + setup_project(ZEN_PORT, extra_zen_args) - for build in BUILDS: - setup_oplog(ZEN_PORT, build.name) + for build in builds: + setup_oplog(ZEN_PORT, build.name, extra_zen_args) print(f"--------- importing {build.name} oplog") - run([ - ZEN_EXE, "oplog-import", + run(zen_cmd( + "oplog-import", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, "--clean", "--builds", "https://jupiter.devtools.epicgames.com", - "--namespace", "fortnite.oplog", + "--namespace", namespace, "--bucket", build.bucket, "--builds-id", build.id, f"--zen-cache-host={ZEN_CACHE}", f"--zen-cache-upload={ZEN_CACHE_POPULATE}", f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}", - ]) + extra_zen_args=extra_zen_args, + )) print() print(f"--------- validating {build.name} oplog") - run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + run(zen_cmd("oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, + extra_zen_args=extra_zen_args)) print() - wipe_or_create("export folder", export_dir) + wipe_or_create("export folder", export_dir, extra_zen_args) - for build in BUILDS: + for build in builds: print(f"--------- exporting {build.name} oplog") - run([ - ZEN_EXE, "oplog-export", + run(zen_cmd( + "oplog-export", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, "--file", export_dir, "--forcetempblocks", - ]) + extra_zen_args=extra_zen_args, + )) print() finally: - stop_server("zenserver", ZEN_PORT) + stop_server("zenserver", ZEN_PORT, extra_zen_args) - wipe_or_create("alternate zenserver data", zen_import_data_dir) - start_server("import zenserver", zen_import_data_dir, ZEN_PORT) + wipe_or_create("alternate zenserver data", zen_import_data_dir, extra_zen_args) + start_server("import zenserver", zen_import_data_dir, ZEN_PORT, extra_zen_args=extra_zen_args) try: - setup_project(ZEN_PORT) + setup_project(ZEN_PORT, extra_zen_args) - for build in BUILDS: - setup_oplog(ZEN_PORT, build.name) + for build in builds: + setup_oplog(ZEN_PORT, build.name, extra_zen_args) print(f"--------- importing {build.name} oplog") - run([ - ZEN_EXE, "oplog-import", + run(zen_cmd( + "oplog-import", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, "--file", export_dir, - ]) + extra_zen_args=extra_zen_args, + )) print() print(f"--------- validating {build.name} oplog") - run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + run(zen_cmd("oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name, + extra_zen_args=extra_zen_args)) print() finally: - stop_server("alternative zenserver", ZEN_PORT) + stop_server("alternative zenserver", ZEN_PORT, extra_zen_args) finally: - stop_server("cache zenserver", ZEN_CACHE_PORT) + stop_server("cache zenserver", ZEN_CACHE_PORT, extra_zen_args) if __name__ == "__main__": diff --git a/scripts/test_scripts/oplog-update-build-ids.py b/scripts/test_scripts/oplog-update-build-ids.py new file mode 100644 index 000000000..67e128c8e --- /dev/null +++ b/scripts/test_scripts/oplog-update-build-ids.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Update oplog-import-export-build-ids.json with build IDs at the highest common changelist across all buckets.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +import tempfile +from pathlib import Path + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" +_DEFAULT_ZEN = Path(f"build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") + + +def _cache_dir() -> Path: + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return base / "Temp" / "zen" + elif sys.platform == "darwin": + return Path.home() / "Library" / "Caches" / "zen" + else: + base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return base / "zen" + + +_OUTPUT_PATH = _cache_dir() / "oplog-import-export-build-ids.json" + +# Maps build name -> Jupiter bucket +_BUILDS: list[tuple[str, str]] = [ + ("XB1Client", "fortnitegame.oplog.fortnite-main.xb1client"), + ("WindowsClient", "fortnitegame.oplog.fortnite-main.windowsclient"), + ("SwitchClient", "fortnitegame.oplog.fortnite-main.switchclient"), + ("XSXClient", "fortnitegame.oplog.fortnite-main.xsxclient"), + ("Switch2Client", "fortnitegame.oplog.fortnite-main.switch2client"), + ("PS4Client", "fortnitegame.oplog.fortnite-main.ps4client"), + ("PS5Client", "fortnitegame.oplog.fortnite-main.ps5client"), + ("LinuxServer", "fortnitegame.oplog.fortnite-main.linuxserver"), + ("IOSClient", "fortnitegame.oplog.fortnite-main.iosclient"), + ("Android_ASTCClient", "fortnitegame.oplog.fortnite-main.android_astcclient"), +] + + +def list_builds_for_bucket(zen: str, host: str, namespace: str, bucket: str) -> list[dict]: + """Run zen builds list for a single bucket and return the results array.""" + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp: + result_path = Path(tmp.name) + + cmd = [ + zen, "builds", "list", + "--namespace", namespace, + "--bucket", bucket, + "--host", host, + "--result-path", str(result_path), + ] + + try: + subprocess.run(cmd, check=True, capture_output=True) + except FileNotFoundError: + sys.exit(f"error: zen binary not found: {zen}") + except subprocess.CalledProcessError as e: + sys.exit( + f"error: zen builds list failed for bucket '{bucket}' with exit code {e.returncode}\n" + f"stderr: {e.stderr.decode(errors='replace')}" + ) + + with result_path.open() as f: + data = json.load(f) + result_path.unlink(missing_ok=True) + + return data.get("results", []) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Refresh oplog-import-export-build-ids.json with build IDs at the highest changelist present in all buckets." + ) + parser.add_argument("--host", default="https://jupiter.devtools.epicgames.com", help="Jupiter host URL") + parser.add_argument("--zen", default=str(_DEFAULT_ZEN), help="Path to the zen binary") + parser.add_argument("--namespace", default="fortnite.oplog", help="Builds storage namespace") + args = parser.parse_args() + + # For each bucket, fetch results and build a changelist -> buildId map. + # bucket_cl_map[bucket] = { changelist_int: buildId_str, ... } + bucket_cl_map: dict[str, dict[int, str]] = {} + + for name, bucket in _BUILDS: + print(f"Querying {name} ({bucket}) ...") + results = list_builds_for_bucket(args.zen, args.host, args.namespace, bucket) + if not results: + sys.exit(f"error: no results for bucket '{bucket}' (build '{name}')") + + cl_map: dict[int, str] = {} + for entry in results: + build_id = entry.get("buildId", "") + metadata = entry.get("metadata") or {} + cl = metadata.get("changelist") + if build_id and cl is not None: + # Keep first occurrence (most recent) per changelist + if cl not in cl_map: + cl_map[int(cl)] = build_id + + if not cl_map: + sys.exit( + f"error: bucket '{bucket}' (build '{name}') returned {len(results)} entries " + "but none had both buildId and changelist in metadata" + ) + + print(f" {len(cl_map)} distinct changelists, latest CL {max(cl_map)}") + bucket_cl_map[bucket] = cl_map + + # Find the highest changelist present in every bucket's result set. + common_cls = set(next(iter(bucket_cl_map.values())).keys()) + for bucket, cl_map in bucket_cl_map.items(): + common_cls &= set(cl_map.keys()) + + if not common_cls: + sys.exit( + "error: no changelist is present in all buckets.\n" + "Per-bucket CL ranges:\n" + + "\n".join( + f" {name} ({bucket}): {min(bucket_cl_map[bucket])} – {max(bucket_cl_map[bucket])}" + for name, bucket in _BUILDS + ) + ) + + best_cl = max(common_cls) + print(f"\nHighest common changelist: {best_cl}") + + build_ids: dict[str, dict[str, str]] = {} + for name, bucket in _BUILDS: + build_id = bucket_cl_map[bucket][best_cl] + build_ids[name] = {"bucket": bucket, "buildId": build_id} + print(f" {name}: {build_id}") + + output = {"namespace": args.namespace, "builds": build_ids} + _OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + with _OUTPUT_PATH.open("w") as f: + json.dump(output, f, indent=2) + f.write("\n") + + print(f"\nWrote {_OUTPUT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/scripts/updatefrontend.lua b/scripts/updatefrontend.lua deleted file mode 100644 index ab37819d7..000000000 --- a/scripts/updatefrontend.lua +++ /dev/null @@ -1,111 +0,0 @@ --- Copyright Epic Games, Inc. All Rights Reserved. - --------------------------------------------------------------------------------- -local function _exec(cmd, ...) - local args = {} - for _, arg in pairs({...}) do - if arg then - table.insert(args, arg) - end - end - - print("--", cmd, table.unpack(args)) - local ret = os.execv(cmd, args) - print() - return ret -end - --------------------------------------------------------------------------------- -local function _zip(store_only, zip_path, ...) - -- Here's the rules; if len(...) is 1 and it is a dir then create a zip with - -- archive paths like this; - -- - -- glob(foo/bar/**) -> foo/bar/abc, foo/bar/dir/123 -> zip(abc, dir/123) - -- - -- Otherwise assume ... is file paths and add without leading directories; - -- - -- foo/abc, bar/123 -> zip(abc, 123) - - zip_path = path.absolute(zip_path) - os.tryrm(zip_path) - - local inputs = {...} - - local source_dir = nil - if #inputs == 1 and os.isdir(inputs[1]) then - source_dir = inputs[1] - end - - import("detect.tools.find_7z") - local cmd_7z = find_7z() - if cmd_7z then - input_paths = {} - if source_dir then - -- Suffixing a directory path with a "/." will have 7z set the path - -- for archived files relative to that directory. - input_paths = { path.join(source_dir, ".") } - else - for _, input_path in pairs(inputs) do - -- If there is a "/./" anywhere in file paths then 7z drops all - -- directory information and just archives the file by name - input_path = path.relative(input_path, ".") - if input_path:sub(2,2) ~= ":" then - input_path = "./"..input_path - end - table.insert(input_paths, input_path) - end - end - - compression_level = "-mx1" - if store_only then - compression_level = "-mx0" - end - - local ret = _exec(cmd_7z, "a", compression_level, zip_path, table.unpack(input_paths)) - if ret > 0 then - raise("Received error from 7z") - end - return - end - - print("7z not found, falling back to zip") - - import("detect.tools.find_zip") - zip_cmd = find_zip() - if zip_cmd then - local input_paths = inputs - local cwd = os.curdir() - if source_dir then - os.cd(source_dir) - input_paths = { "." } - end - - compression_level = "-1" - if store_only then - compression_level = "-0" - end - - local strip_leading_path = nil - if not source_dir then - strip_leading_path = "--junk-paths" - end - - local ret = _exec(zip_cmd, "-r", compression_level, strip_leading_path, zip_path, table.unpack(input_paths)) - if ret > 0 then - raise("Received error from zip") - end - - os.cd(cwd) - return - end - print("zip not found") - - raise("Unable to find a suitable zip tool") -end - --------------------------------------------------------------------------------- -function main() - local zip_path = "src/zenserver/frontend/html.zip" - local content_dir = "src/zenserver/frontend/html/" - _zip(true, zip_path, content_dir) -end diff --git a/src/transports/winsock/winsock.cpp b/src/transports/winsock/winsock.cpp index f98984726..c1f4f6abe 100644 --- a/src/transports/winsock/winsock.cpp +++ b/src/transports/winsock/winsock.cpp @@ -271,7 +271,7 @@ WinsockTransportPlugin::Initialize(TransportServer* ServerInterface) m_ServerInterface = ServerInterface; m_ListenSocket = socket(AF_INET6, SOCK_STREAM, 0); - if (m_ListenSocket == SOCKET_ERROR || m_ListenSocket == INVALID_SOCKET) + if (m_ListenSocket == INVALID_SOCKET) { throw std::system_error(std::error_code(WSAGetLastError(), std::system_category()), "socket creation failed in HTTP plugin server init"); @@ -302,7 +302,7 @@ WinsockTransportPlugin::Initialize(TransportServer* ServerInterface) do { - if (SOCKET ClientSocket = accept(m_ListenSocket, NULL, NULL); ClientSocket != SOCKET_ERROR) + if (SOCKET ClientSocket = accept(m_ListenSocket, NULL, NULL); ClientSocket != INVALID_SOCKET) { int Flag = 1; setsockopt(ClientSocket, IPPROTO_TCP, TCP_NODELAY, (char*)&Flag, sizeof(Flag)); diff --git a/src/zencore/include/zencore/logbase.h b/src/zencore/include/zencore/logbase.h index ad2ab218d..046e96db3 100644 --- a/src/zencore/include/zencore/logbase.h +++ b/src/zencore/include/zencore/logbase.h @@ -101,7 +101,7 @@ struct LoggerRef inline logging::Logger* operator->() const; inline logging::Logger& operator*() const; - bool ShouldLog(logging::LogLevel Level) const { return m_Logger->ShouldLog(Level); } + bool ShouldLog(logging::LogLevel Level) const { return m_Logger && m_Logger->ShouldLog(Level); } void SetLogLevel(logging::LogLevel NewLogLevel) { m_Logger->SetLevel(NewLogLevel); } logging::LogLevel GetLogLevel() { return m_Logger->GetLevel(); } diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index 5a8a66fae..7b46f0e38 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -331,7 +331,7 @@ public: return AppendAscii(Str); } -#if defined(__clang__) && !defined(__APPLE__) +#if defined(__clang__) && !defined(__APPLE__) && !defined(_MSC_VER) /* UE Toolchain Clang has different types for int64_t and long long so an override is needed here. Without it, Clang can't disambiguate integer overloads */ inline StringBuilderImpl& operator<<(long long n) diff --git a/src/zencore/memtrack/callstacktrace.cpp b/src/zencore/memtrack/callstacktrace.cpp index 4a7068568..013c51535 100644 --- a/src/zencore/memtrack/callstacktrace.cpp +++ b/src/zencore/memtrack/callstacktrace.cpp @@ -912,92 +912,100 @@ FBacktracer::GetBacktraceId(void* AddressOfReturnAddress) # else // UE_CALLSTACK_TRACE_USE_UNWIND_TABLES -namespace zen { +//////////////////////////////////////////////////////////////////////////////// +class FBacktracer +{ +public: + FBacktracer(FMalloc* InMalloc); + ~FBacktracer(); + static FBacktracer* Get(); + inline uint32_t GetBacktraceId(void* AddressOfReturnAddress); + uint32_t GetBacktraceId(uint64_t ReturnAddress); + void AddModule(uintptr_t Base, const char16_t* Name) {} + void RemoveModule(uintptr_t Base) {} - //////////////////////////////////////////////////////////////////////////////// - class FBacktracer - { - public: - FBacktracer(FMalloc* InMalloc); - ~FBacktracer(); - static FBacktracer* Get(); - inline uint32_t GetBacktraceId(void* AddressOfReturnAddress); - uint32_t GetBacktraceId(uint64_t ReturnAddress); - void AddModule(uintptr_t Base, const char16_t* Name) {} - void RemoveModule(uintptr_t Base) {} - - private: - static FBacktracer* Instance; - FMalloc* Malloc; - FCallstackTracer CallstackTracer; - }; +private: + static FBacktracer* Instance; + FMalloc* Malloc; + FCallstackTracer CallstackTracer; +}; - //////////////////////////////////////////////////////////////////////////////// - FBacktracer* FBacktracer::Instance = nullptr; +//////////////////////////////////////////////////////////////////////////////// +FBacktracer* FBacktracer::Instance = nullptr; - //////////////////////////////////////////////////////////////////////////////// - FBacktracer::FBacktracer(FMalloc* InMalloc) : Malloc(InMalloc), CallstackTracer(InMalloc) { Instance = this; } +//////////////////////////////////////////////////////////////////////////////// +FBacktracer::FBacktracer(FMalloc* InMalloc) : Malloc(InMalloc), CallstackTracer(InMalloc) +{ + Instance = this; +} - //////////////////////////////////////////////////////////////////////////////// - FBacktracer::~FBacktracer() {} +//////////////////////////////////////////////////////////////////////////////// +FBacktracer::~FBacktracer() +{ +} - //////////////////////////////////////////////////////////////////////////////// - FBacktracer* FBacktracer::Get() { return Instance; } +//////////////////////////////////////////////////////////////////////////////// +FBacktracer* +FBacktracer::Get() +{ + return Instance; +} - //////////////////////////////////////////////////////////////////////////////// - uint32_t FBacktracer::GetBacktraceId(void* AddressOfReturnAddress) - { - const uint64_t ReturnAddress = *(uint64_t*)AddressOfReturnAddress; - return GetBacktraceId(ReturnAddress); - } +//////////////////////////////////////////////////////////////////////////////// +uint32_t +FBacktracer::GetBacktraceId(void* AddressOfReturnAddress) +{ + const uint64_t ReturnAddress = *(uint64_t*)AddressOfReturnAddress; + return GetBacktraceId(ReturnAddress); +} - //////////////////////////////////////////////////////////////////////////////// - uint32_t FBacktracer::GetBacktraceId(uint64_t ReturnAddress) - { +//////////////////////////////////////////////////////////////////////////////// +uint32_t +FBacktracer::GetBacktraceId(uint64_t ReturnAddress) +{ # if !UE_BUILD_SHIPPING - uint64_t StackFrames[256]; - int32_t NumStackFrames = FPlatformStackWalk::CaptureStackBackTrace(StackFrames, UE_ARRAY_COUNT(StackFrames)); - if (NumStackFrames > 0) + uint64_t StackFrames[256]; + int32_t NumStackFrames = FPlatformStackWalk::CaptureStackBackTrace(StackFrames, UE_ARRAY_COUNT(StackFrames)); + if (NumStackFrames > 0) + { + FCallstackTracer::FBacktraceEntry BacktraceEntry; + uint64_t BacktraceId = 0; + uint32_t FrameIdx = 0; + bool bUseAddress = false; + for (int32_t Index = 0; Index < NumStackFrames; Index++) { - FCallstackTracer::FBacktraceEntry BacktraceEntry; - uint64_t BacktraceId = 0; - uint32_t FrameIdx = 0; - bool bUseAddress = false; - for (int32_t Index = 0; Index < NumStackFrames; Index++) + if (!bUseAddress) { - if (!bUseAddress) + // start using backtrace only after ReturnAddress + if (StackFrames[Index] == (uint64_t)ReturnAddress) { - // start using backtrace only after ReturnAddress - if (StackFrames[Index] == (uint64_t)ReturnAddress) - { - bUseAddress = true; - } - } - if (bUseAddress || NumStackFrames == 1) - { - uint64_t RetAddr = StackFrames[Index]; - StackFrames[FrameIdx++] = RetAddr; - - // This is a simple order-dependent LCG. Should be sufficient enough - BacktraceId += RetAddr; - BacktraceId *= 0x30be8efa499c249dull; + bUseAddress = true; } } + if (bUseAddress || NumStackFrames == 1) + { + uint64_t RetAddr = StackFrames[Index]; + StackFrames[FrameIdx++] = RetAddr; - // Save the collected id - BacktraceEntry.Hash = BacktraceId; - BacktraceEntry.FrameCount = FrameIdx; - BacktraceEntry.Frames = StackFrames; - - // Add to queue to be processed. This might block until there is room in the - // queue (i.e. the processing thread has caught up processing). - return CallstackTracer.AddCallstack(BacktraceEntry); + // This is a simple order-dependent LCG. Should be sufficient enough + BacktraceId += RetAddr; + BacktraceId *= 0x30be8efa499c249dull; + } } -# endif - return 0; + // Save the collected id + BacktraceEntry.Hash = BacktraceId; + BacktraceEntry.FrameCount = FrameIdx; + BacktraceEntry.Frames = StackFrames; + + // Add to queue to be processed. This might block until there is room in the + // queue (i.e. the processing thread has caught up processing). + return CallstackTracer.AddCallstack(BacktraceEntry); } +# endif + return 0; +} } # endif // UE_CALLSTACK_TRACE_USE_UNWIND_TABLES @@ -1047,7 +1055,7 @@ CallstackTrace_GetCurrentId() # if PLATFORM_USE_CALLSTACK_ADDRESS_POINTER return Instance->GetBacktraceId(StackAddress); # else - return Instance->GetBacktraceId((uint64_t)StackAddress); + return Instance->GetBacktraceId((uint64_t)StackAddress); # endif } diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index deeeb6c85..9f49802a0 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -36,15 +36,17 @@ namespace zen { +#if ZEN_WITH_CPR extern HttpClientBase* CreateCprHttpClient(std::string_view BaseUri, const HttpClientSettings& ConnectionSettings, std::function<bool()>&& CheckIfAbortFunction); +#endif extern HttpClientBase* CreateCurlHttpClient(std::string_view BaseUri, const HttpClientSettings& ConnectionSettings, std::function<bool()>&& CheckIfAbortFunction); -static HttpClientBackend g_DefaultHttpClientBackend = HttpClientBackend::kCpr; +static HttpClientBackend g_DefaultHttpClientBackend = HttpClientBackend::kCurl; void SetDefaultHttpClientBackend(HttpClientBackend Backend) @@ -55,11 +57,14 @@ SetDefaultHttpClientBackend(HttpClientBackend Backend) void SetDefaultHttpClientBackend(std::string_view Backend) { +#if ZEN_WITH_CPR if (Backend == "cpr") { g_DefaultHttpClientBackend = HttpClientBackend::kCpr; } - else if (Backend == "curl") + else +#endif + if (Backend == "curl") { g_DefaultHttpClientBackend = HttpClientBackend::kCurl; } @@ -363,13 +368,15 @@ HttpClient::HttpClient(std::string_view BaseUri, const HttpClientSettings& Conne switch (EffectiveBackend) { - case HttpClientBackend::kCurl: - m_Inner = CreateCurlHttpClient(BaseUri, ConnectionSettings, std::move(CheckIfAbortFunction)); - break; +#if ZEN_WITH_CPR case HttpClientBackend::kCpr: - default: m_Inner = CreateCprHttpClient(BaseUri, ConnectionSettings, std::move(CheckIfAbortFunction)); break; +#endif + case HttpClientBackend::kCurl: + default: + m_Inner = CreateCurlHttpClient(BaseUri, ConnectionSettings, std::move(CheckIfAbortFunction)); + break; } } diff --git a/src/zenhttp/include/zenhttp/cprutils.h b/src/zenhttp/include/zenhttp/cprutils.h index c252a5d99..3cfe652c5 100644 --- a/src/zenhttp/include/zenhttp/cprutils.h +++ b/src/zenhttp/include/zenhttp/cprutils.h @@ -2,17 +2,19 @@ #pragma once -#include <zencore/compactbinary.h> -#include <zencore/compactbinaryvalidation.h> -#include <zencore/iobuffer.h> -#include <zencore/string.h> -#include <zenhttp/formatters.h> -#include <zenhttp/httpclient.h> -#include <zenhttp/httpcommon.h> +#if ZEN_WITH_CPR + +# include <zencore/compactbinary.h> +# include <zencore/compactbinaryvalidation.h> +# include <zencore/iobuffer.h> +# include <zencore/string.h> +# include <zenhttp/formatters.h> +# include <zenhttp/httpclient.h> +# include <zenhttp/httpcommon.h> ZEN_THIRD_PARTY_INCLUDES_START -#include <cpr/response.h> -#include <fmt/format.h> +# include <cpr/response.h> +# include <fmt/format.h> ZEN_THIRD_PARTY_INCLUDES_END template<> @@ -92,3 +94,5 @@ struct fmt::formatter<cpr::Response> } } }; + +#endif // ZEN_WITH_CPR diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index e95d78772..e878c900f 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -52,7 +52,9 @@ enum class HttpClientErrorCode : int enum class HttpClientBackend : uint8_t { kDefault, +#if ZEN_WITH_CPR kCpr, +#endif kCurl, }; diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index 4bf8c61bb..a1bb719c8 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -147,7 +147,7 @@ public: HttpPluginServerRequest& operator=(const HttpPluginServerRequest&) = delete; // As this is plugin transport connection used for specialized connections we assume it is not a machine local connection - virtual bool IsLocalMachineRequest() const /* override*/ { return false; } + bool IsLocalMachineRequest() const override { return false; } virtual std::string_view GetAuthorizationHeader() const override; virtual Oid ParseSessionId() const override; virtual uint32_t ParseRequestId() const override; diff --git a/src/zenhttp/xmake.lua b/src/zenhttp/xmake.lua index 9b461662e..b4c65ea96 100644 --- a/src/zenhttp/xmake.lua +++ b/src/zenhttp/xmake.lua @@ -8,7 +8,12 @@ target('zenhttp') add_files("servers/httpsys.cpp", {unity_ignored=true}) add_files("servers/wshttpsys.cpp", {unity_ignored=true}) add_includedirs("include", {public=true}) - add_deps("zencore", "zentelemetry", "transport-sdk", "asio", "cpr") + add_deps("zencore", "zentelemetry", "transport-sdk", "asio") + if has_config("zencpr") then + add_deps("cpr") + else + remove_files("clients/httpclientcpr.cpp") + end add_packages("http_parser", "json11") add_options("httpsys") diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip Binary files differdeleted file mode 100644 index 58778a592..000000000 --- a/src/zenserver/frontend/html.zip +++ /dev/null diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index 3cfaa956d..52889fff5 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -12,12 +12,14 @@ target("zenserver") "zenremotestore", "zenstore", "zentelemetry", - "zenutil", - "zenvfs") + "zenutil") + if is_plat("windows") then + add_deps("zenvfs") + end add_headerfiles("**.h") add_rules("utils.bin2c", {extensions = {".zip"}}) add_files("**.cpp") - add_files("frontend/*.zip") + add_files("frontend/html.zip") add_files("zenserver.cpp", {unity_ignored = true }) if is_plat("linux") and not (get_config("toolchain") or ""):find("clang") then @@ -78,7 +80,45 @@ target("zenserver") add_ldflags("-framework SystemConfiguration") end - -- to work around some unfortunate Ctrl-C behaviour on Linux/Mac due to + on_load(function(target) + local html_dir = path.join(os.projectdir(), "src/zenserver/frontend/html") + local zip_path = path.join(os.projectdir(), "src/zenserver/frontend/html.zip") + + -- Check if zip needs regeneration + local need_update = not os.isfile(zip_path) + if not need_update then + local zip_mtime = os.mtime(zip_path) + for _, file in ipairs(os.files(path.join(html_dir, "**"))) do + if os.mtime(file) > zip_mtime then + need_update = true + break + end + end + end + + if need_update then + print("Regenerating frontend zip...") + os.tryrm(zip_path) + + import("detect.tools.find_7z") + local cmd_7z = find_7z() + if cmd_7z then + os.execv(cmd_7z, {"a", "-mx0", zip_path, path.join(html_dir, ".")}) + else + import("detect.tools.find_zip") + local zip_cmd = find_zip() + if zip_cmd then + local oldir = os.cd(html_dir) + os.execv(zip_cmd, {"-r", "-0", zip_path, "."}) + os.cd(oldir) + else + raise("Unable to find a suitable zip tool (need 7z or zip)") + end + end + end + end) + + -- to work around some unfortunate Ctrl-C behaviour on Linux/Mac due to -- our use of setsid() at startup we pass in `--no-detach` to zenserver -- ensure that it recieves signals when the user requests termination on_run(function(target) diff --git a/src/zenstore/xmake.lua b/src/zenstore/xmake.lua index ea8155e94..94c2b51ca 100644 --- a/src/zenstore/xmake.lua +++ b/src/zenstore/xmake.lua @@ -6,6 +6,11 @@ target('zenstore') add_headerfiles("**.h") add_files("**.cpp") add_includedirs("include", {public=true}) - add_deps("zencore", "zentelemetry", "zenutil", "zenvfs") + add_deps("zencore", "zentelemetry", "zenutil") + if is_plat("windows") then + add_deps("zenvfs") + else + add_includedirs("$(projectdir)/src/zenvfs/include", {public=true}) + end add_deps("robin-map") add_packages("eastl", {public=true}); diff --git a/src/zenutil/workerpools.cpp b/src/zenutil/workerpools.cpp index 1bab39b2a..25f961f77 100644 --- a/src/zenutil/workerpools.cpp +++ b/src/zenutil/workerpools.cpp @@ -25,9 +25,9 @@ namespace { struct WorkerPool { - std::unique_ptr<WorkerThreadPool> Pool; const int TreadCount; const std::string_view Name; + std::unique_ptr<WorkerThreadPool> Pool; }; WorkerPool BurstLargeWorkerPool = {.TreadCount = LargeWorkerThreadPoolTreadCount, .Name = "large"}; diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua index 3e8d88c5d..013ed8de3 100644 --- a/thirdparty/xmake.lua +++ b/thirdparty/xmake.lua @@ -37,7 +37,7 @@ target('rpmalloc') set_group('thirdparty') set_languages("c17", "cxx20") if is_os("windows") then - add_cflags("/experimental:c11atomics", {force=true}) + add_cflags("/experimental:c11atomics", {force=true, tools="cl"}) end add_defines("RPMALLOC_FIRST_CLASS_HEAPS=1", "ENABLE_STATISTICS=1", "ENABLE_OVERRIDE=0") add_files("rpmalloc/rpmalloc.c") @@ -84,7 +84,7 @@ target("blake3") add_includedirs("blake3/c", {public=true}) if is_os("windows") then - add_cflags("/experimental:c11atomics") + add_cflags("/experimental:c11atomics", {tools="cl"}) add_cflags("/wd4245", {force = true}) -- conversion from 'type1' to 'type2', possible loss of data elseif is_os("macosx") then add_cflags("-Wno-unused-function") @@ -191,6 +191,30 @@ if is_os("linux") or is_os("macosx") then add_cxxflags("-Wno-unused-but-set-variable", {tools="gcc"}) end +if get_config("toolchain") == "clang-cl" then + add_cxxflags("-Wno-unknown-warning-option", {force=true}) + add_cxxflags("-Wno-cast-function-type-mismatch", {force=true}) + add_cxxflags("-Wno-delete-non-abstract-non-virtual-dtor", {force=true}) + add_cxxflags("-Wno-format", {force=true}) + add_cxxflags("-Wno-implicit-fallthrough", {force=true}) + add_cxxflags("-Wno-inconsistent-missing-override", {force=true}) + add_cxxflags("-Wno-missing-field-initializers", {force=true}) + add_cxxflags("-Wno-parentheses-equality", {force=true}) + add_cxxflags("-Wno-reorder-ctor", {force=true}) + add_cxxflags("-Wno-sign-compare", {force=true}) + add_cxxflags("-Wno-strict-aliasing", {force=true}) + add_cxxflags("-Wno-switch", {force=true}) + add_cxxflags("-Wno-unused-but-set-variable", {force=true}) + add_cxxflags("-Wno-unused-lambda-capture", {force=true}) + add_cxxflags("-Wno-unused-parameter", {force=true}) + add_cxxflags("-Wno-unused-private-field", {force=true}) + add_cxxflags("-Wno-unused-value", {force=true}) + add_cxxflags("-Wno-unused-variable", {force=true}) + add_cxxflags("-Wno-vla-cxx-extension", {force=true}) + add_cflags("-Wno-unknown-warning-option", {force=true}) + add_cflags("-Wno-unused-command-line-argument", {force=true}) +end + if is_os("macosx") then -- silence warnings about -Wno-vla-cxx-extension since to my knowledge we can't -- detect the clang version used in Xcode and only recent versions contain this flag @@ -288,6 +312,13 @@ option("zentrace") option_end() add_define_by_config("ZEN_WITH_TRACE", "zentrace") +option("zencpr") + set_default(true) + set_showmenu(true) + set_description("Enable CPR HTTP client backend") +option_end() +add_define_by_config("ZEN_WITH_CPR", "zencpr") + set_warnings("allextra", "error") set_languages("cxx20") @@ -330,7 +361,9 @@ end includes("src/zenstore", "src/zenstore-test") includes("src/zentelemetry", "src/zentelemetry-test") includes("src/zenutil", "src/zenutil-test") -includes("src/zenvfs") +if is_plat("windows") then + includes("src/zenvfs") +end includes("src/zenserver", "src/zenserver-test") includes("src/zen") includes("src/zentest-appstub") @@ -368,16 +401,6 @@ task("kill") end end) -task("updatefrontend") - set_menu { - usage = "xmake updatefrontend", - description = "Create Zip of the frontend/html folder for bundling with zenserver executable", - } - on_run(function() - import("scripts.updatefrontend") - updatefrontend() - end) - task("precommit") set_menu { usage = "xmake precommit", |