aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2026-03-15 22:53:37 +0100
committerStefan Boberg <[email protected]>2026-03-15 22:53:37 +0100
commitda216166aab918bc93ee73040c1eb6336ea6d026 (patch)
tree87263e1e70d975c640981a50474290cdad1382cc
parentChange --noclean to --clean in toolchain verify script (diff)
parentMerge branch 'main' into sb/linux-build-improvements (diff)
downloadzen-sb/linux-build-improvements.tar.xz
zen-sb/linux-build-improvements.zip
Merge branch 'sb/linux-build-improvements' of ssh://arn-wd-l1704.localdomain:2222/ue-foundation/zen into sb/linux-build-improvementssb/linux-build-improvements
-rw-r--r--.gitignore3
-rw-r--r--CLAUDE.md4
-rw-r--r--scripts/test_scripts/builds-download-upload-test.py74
-rw-r--r--scripts/test_scripts/builds-download-upload-update-build-ids.py150
-rw-r--r--scripts/test_scripts/oplog-import-export-test.py177
-rw-r--r--scripts/test_scripts/oplog-update-build-ids.py151
-rw-r--r--scripts/updatefrontend.lua111
-rw-r--r--src/transports/winsock/winsock.cpp4
-rw-r--r--src/zencore/include/zencore/logbase.h2
-rw-r--r--src/zencore/include/zencore/string.h2
-rw-r--r--src/zencore/memtrack/callstacktrace.cpp146
-rw-r--r--src/zenhttp/httpclient.cpp19
-rw-r--r--src/zenhttp/include/zenhttp/cprutils.h22
-rw-r--r--src/zenhttp/include/zenhttp/httpclient.h2
-rw-r--r--src/zenhttp/servers/httpplugin.cpp2
-rw-r--r--src/zenhttp/xmake.lua7
-rw-r--r--src/zenserver/frontend/html.zipbin431365 -> 0 bytes
-rw-r--r--src/zenserver/xmake.lua48
-rw-r--r--src/zenstore/xmake.lua7
-rw-r--r--src/zenutil/workerpools.cpp2
-rw-r--r--thirdparty/xmake.lua4
-rw-r--r--xmake.lua45
22 files changed, 685 insertions, 297 deletions
diff --git a/.gitignore b/.gitignore
index 1073d5513..5c9195566 100644
--- a/.gitignore
+++ b/.gitignore
@@ -113,3 +113,6 @@ CMake*
# Ue tool chain temp directory
.tmp-ue-toolchain/
+
+# Generated frontend zip (built automatically by xmake)
+src/zenserver/frontend/html.zip
diff --git a/CLAUDE.md b/CLAUDE.md
index 32c4b3d40..45a9bddce 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -173,7 +173,7 @@ The codebase is organized into layered modules with clear dependencies:
- Web UI bundled as ZIP in `src/zenserver/frontend/*.zip`
- Dashboards for hub, orchestrator, and compute services are located in `src/zenserver/frontent/html/`
- These are the files which end up being bundled into the front-end zip mentioned above
-- Update with `xmake updatefrontend` after modifying HTML/JS, and check in the resulting zip
+- The zip is generated automatically at build time when source files change
**Memory Management:**
- Can use mimalloc or rpmalloc for performance
@@ -309,6 +309,4 @@ When debugging zenserver-test or other multi-process scenarios, use child proces
# Create deployable ZIP bundle
xmake bundle
-# Update frontend ZIP after HTML changes
-xmake updatefrontend
```
diff --git a/scripts/test_scripts/builds-download-upload-test.py b/scripts/test_scripts/builds-download-upload-test.py
index e4fee7cb8..8ff5245c1 100644
--- a/scripts/test_scripts/builds-download-upload-test.py
+++ b/scripts/test_scripts/builds-download-upload-test.py
@@ -4,6 +4,8 @@
from __future__ import annotations
import argparse
+import json
+import os
import platform
import subprocess
import sys
@@ -15,22 +17,51 @@ _ARCH = "x64" if sys.platform == "win32" else platform.machine().lower()
_EXE_SUFFIX = ".exe" if sys.platform == "win32" else ""
+def _cache_dir() -> Path:
+ if sys.platform == "win32":
+ base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+ return base / "Temp" / "zen"
+ elif sys.platform == "darwin":
+ return Path.home() / "Library" / "Caches" / "zen"
+ else:
+ base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+ return base / "zen"
+
+
+_BUILD_IDS_PATH = _cache_dir() / "builds-download-upload-build-ids.json"
+
+
class Build(NamedTuple):
name: str
bucket: str
id: str
-BUILDS = [
- Build("XB1Client", "fortnitegame.staged-build.fortnite-main.xb1-client", "09a7616c1a388dfe6056aa57"),
- Build("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client", "09a762c81e2cf213142d0ce5"),
- Build("SwitchClient", "fortnitegame.staged-build.fortnite-main.switch-client", "09a75bf9c3ce75bce09f644f"),
- Build("LinuxServer", "fortnitegame.staged-build.fortnite-main.linux-server", "09a750ac155eb3e3b62e87e0"),
- Build("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client", "09a78f3df07b289691ec5710"),
- Build("PS4Client", "fortnitegame.staged-build.fortnite-main.ps4-client", "09a76ea92ad301d4724fafad"),
- Build("IOSClient", "fortnitegame.staged-build.fortnite-main.ios-client", "09a7816fa26c23362fef0c5d"),
- Build("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client", "09a76725f1620d62c6be06e4"),
-]
+def load_builds() -> tuple[str, list[Build]]:
+ if not _BUILD_IDS_PATH.exists():
+ print(f"Build IDs file not found: {_BUILD_IDS_PATH}")
+ answer = input("Run builds-download-upload-update-build-ids.py now to populate it? [y/N] ").strip().lower()
+ if answer == "y":
+ update_script = Path(__file__).parent / "builds-download-upload-update-build-ids.py"
+ subprocess.run([sys.executable, str(update_script)], check=True)
+ else:
+ sys.exit("Aborted. Run scripts/test_scripts/builds-download-upload-update-build-ids.py to populate it.")
+ with _BUILD_IDS_PATH.open() as f:
+ data: dict = json.load(f)
+ namespace = data.get("namespace", "")
+ if not namespace:
+ sys.exit(f"error: {_BUILD_IDS_PATH} is missing 'namespace'")
+ builds = []
+ for name, entry in data.get("builds", {}).items():
+ bucket = entry.get("bucket", "")
+ build_id = entry.get("buildId", "")
+ if not bucket or not build_id:
+ sys.exit(f"error: entry '{name}' in {_BUILD_IDS_PATH} is missing 'bucket' or 'buildId'")
+ builds.append(Build(name, bucket, build_id))
+ if not builds:
+ sys.exit(f"error: {_BUILD_IDS_PATH} contains no builds")
+ return namespace, builds
+
ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}")
ZEN_METADATA_DIR: Path = Path(__file__).resolve().parent / "metadatas"
@@ -99,12 +130,12 @@ def wipe_or_create(label: str, path: Path, extra_zen_args: list[str] | None = No
print()
-def check_prerequisites() -> None:
+def check_prerequisites(builds: list[Build]) -> None:
if not ZEN_EXE.is_file():
sys.exit(f"error: zen executable not found: {ZEN_EXE}")
if not ZEN_METADATA_DIR.is_dir():
sys.exit(f"error: metadata directory not found: {ZEN_METADATA_DIR}")
- for build in BUILDS:
+ for build in builds:
metadata = ZEN_METADATA_DIR / f"{build.name}.json"
if not metadata.is_file():
sys.exit(f"error: metadata file not found: {metadata}")
@@ -145,10 +176,10 @@ def main() -> None:
)
parser.add_argument(
"--data-path",
- default=Path(Path(__file__).stem + "_datadir"),
+ default=None,
type=Path,
metavar="PATH",
- help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)",
+ help="root path for all data directories",
)
parser.add_argument(
"--zen-exe-path",
@@ -162,17 +193,24 @@ def main() -> None:
data_path = args.positional_path
if data_path is None:
data_path = args.data_path
+ if data_path is None:
+ print("WARNING: This script may require up to 1TB of free disk space.")
+ raw = input("Enter root path for all data directories: ").strip()
+ if not raw:
+ sys.exit("error: data path is required")
+ data_path = Path(raw)
ZEN_EXE = args.zen_exe_positional
if ZEN_EXE is None:
ZEN_EXE = args.zen_exe_path
+ namespace, builds = load_builds()
zen_system_dir = data_path / "system"
zen_download_dir = data_path / "Download"
zen_cache_data_dir = data_path / "ZenBuildsCache"
zen_upload_dir = data_path / "Upload"
zen_chunk_cache_path = data_path / "ChunkCache"
- check_prerequisites()
+ check_prerequisites(builds)
start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT,
extra_zen_args=extra_zen_args, extra_server_args=["--buildstore-enabled"])
@@ -180,12 +218,12 @@ def main() -> None:
wipe_or_create("download folder", zen_download_dir, extra_zen_args)
wipe_or_create("system folder", zen_system_dir, extra_zen_args)
- for build in BUILDS:
+ for build in builds:
print(f"--------- importing {build.name} build")
run(zen_cmd(
"builds", "download",
"--host", "https://jupiter.devtools.epicgames.com",
- "--namespace", "fortnite.oplog",
+ "--namespace", namespace,
"--bucket", build.bucket,
"--build-id", build.id,
"--local-path", zen_download_dir / build.name,
@@ -199,7 +237,7 @@ def main() -> None:
wipe_or_create("upload folder", zen_upload_dir, extra_zen_args)
- for build in BUILDS:
+ for build in builds:
print(f"--------- exporting {build.name} build")
run(zen_cmd(
"builds", "upload",
diff --git a/scripts/test_scripts/builds-download-upload-update-build-ids.py b/scripts/test_scripts/builds-download-upload-update-build-ids.py
new file mode 100644
index 000000000..2a63aa44d
--- /dev/null
+++ b/scripts/test_scripts/builds-download-upload-update-build-ids.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Update builds-download-upload-build-ids.json with build IDs at the highest common changelist across all buckets."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import platform
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux"
+_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower()
+_EXE_SUFFIX = ".exe" if sys.platform == "win32" else ""
+_DEFAULT_ZEN = Path(f"build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}")
+
+
+def _cache_dir() -> Path:
+ if sys.platform == "win32":
+ base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+ return base / "Temp" / "zen"
+ elif sys.platform == "darwin":
+ return Path.home() / "Library" / "Caches" / "zen"
+ else:
+ base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+ return base / "zen"
+
+
+_OUTPUT_PATH = _cache_dir() / "builds-download-upload-build-ids.json"
+
+# Maps build name -> Jupiter bucket
+_BUILDS: list[tuple[str, str]] = [
+ ("XB1Client", "fortnitegame.staged-build.fortnite-main.xb1-client"),
+ ("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client"),
+ ("SwitchClient", "fortnitegame.staged-build.fortnite-main.switch-client"),
+ ("LinuxServer", "fortnitegame.staged-build.fortnite-main.linux-server"),
+ ("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client"),
+ ("PS4Client", "fortnitegame.staged-build.fortnite-main.ps4-client"),
+ ("PS5Client", "fortnitegame.staged-build.fortnite-main.ps5-client"),
+ ("IOSClient", "fortnitegame.staged-build.fortnite-main.ios-client"),
+ ("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client"),
+]
+
+
+def list_builds_for_bucket(zen: str, host: str, namespace: str, bucket: str) -> list[dict]:
+ """Run zen builds list for a single bucket and return the results array."""
+ with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp:
+ result_path = Path(tmp.name)
+
+ cmd = [
+ zen, "builds", "list",
+ "--namespace", namespace,
+ "--bucket", bucket,
+ "--host", host,
+ "--result-path", str(result_path),
+ ]
+
+ try:
+ subprocess.run(cmd, check=True, capture_output=True)
+ except FileNotFoundError:
+ sys.exit(f"error: zen binary not found: {zen}")
+ except subprocess.CalledProcessError as e:
+ sys.exit(
+ f"error: zen builds list failed for bucket '{bucket}' with exit code {e.returncode}\n"
+ f"stderr: {e.stderr.decode(errors='replace')}"
+ )
+
+ with result_path.open() as f:
+ data = json.load(f)
+ result_path.unlink(missing_ok=True)
+
+ return data.get("results", [])
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(
+ description="Refresh builds-download-upload-build-ids.json with build IDs at the highest changelist present in all buckets."
+ )
+ parser.add_argument("--host", default="https://jupiter.devtools.epicgames.com", help="Jupiter host URL")
+ parser.add_argument("--zen", default=str(_DEFAULT_ZEN), help="Path to the zen binary")
+ parser.add_argument("--namespace", default="fortnite.oplog", help="Builds storage namespace")
+ args = parser.parse_args()
+
+ # For each bucket, fetch results and build a changelist -> buildId map.
+ # bucket_cl_map[bucket] = { changelist_int: buildId_str, ... }
+ bucket_cl_map: dict[str, dict[int, str]] = {}
+
+ for name, bucket in _BUILDS:
+ print(f"Querying {name} ({bucket}) ...")
+ results = list_builds_for_bucket(args.zen, args.host, args.namespace, bucket)
+ if not results:
+ sys.exit(f"error: no results for bucket '{bucket}' (build '{name}')")
+
+ cl_map: dict[int, str] = {}
+ for entry in results:
+ build_id = entry.get("buildId", "")
+ metadata = entry.get("metadata") or {}
+ cl = metadata.get("commit")
+ if build_id and cl is not None:
+ # Keep first occurrence (most recent) per changelist
+ if cl not in cl_map:
+ cl_map[int(cl)] = build_id
+
+ if not cl_map:
+ sys.exit(
+ f"error: bucket '{bucket}' (build '{name}') returned {len(results)} entries "
+ "but none had both buildId and changelist in metadata"
+ )
+
+ print(f" {len(cl_map)} distinct changelists, latest CL {max(cl_map)}")
+ bucket_cl_map[bucket] = cl_map
+
+ # Find the highest changelist present in every bucket's result set.
+ common_cls = set(next(iter(bucket_cl_map.values())).keys())
+ for bucket, cl_map in bucket_cl_map.items():
+ common_cls &= set(cl_map.keys())
+
+ if not common_cls:
+ sys.exit(
+ "error: no changelist is present in all buckets.\n"
+ "Per-bucket CL ranges:\n"
+ + "\n".join(
+ f" {name} ({bucket}): {min(bucket_cl_map[bucket])} – {max(bucket_cl_map[bucket])}"
+ for name, bucket in _BUILDS
+ )
+ )
+
+ best_cl = max(common_cls)
+ print(f"\nHighest common changelist: {best_cl}")
+
+ build_ids: dict[str, dict[str, str]] = {}
+ for name, bucket in _BUILDS:
+ build_id = bucket_cl_map[bucket][best_cl]
+ build_ids[name] = {"bucket": bucket, "buildId": build_id}
+ print(f" {name}: {build_id}")
+
+ output = {"namespace": args.namespace, "builds": build_ids}
+ _OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+ with _OUTPUT_PATH.open("w") as f:
+ json.dump(output, f, indent=2)
+ f.write("\n")
+
+ print(f"\nWrote {_OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/test_scripts/oplog-import-export-test.py b/scripts/test_scripts/oplog-import-export-test.py
index b2a5ece6c..f913a7351 100644
--- a/scripts/test_scripts/oplog-import-export-test.py
+++ b/scripts/test_scripts/oplog-import-export-test.py
@@ -4,6 +4,8 @@
from __future__ import annotations
import argparse
+import json
+import os
import platform
import subprocess
import sys
@@ -15,23 +17,51 @@ _ARCH = "x64" if sys.platform == "win32" else platform.machine().lower()
_EXE_SUFFIX = ".exe" if sys.platform == "win32" else ""
+def _cache_dir() -> Path:
+ if sys.platform == "win32":
+ base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+ return base / "Temp" / "zen"
+ elif sys.platform == "darwin":
+ return Path.home() / "Library" / "Caches" / "zen"
+ else:
+ base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+ return base / "zen"
+
+
+_BUILD_IDS_PATH = _cache_dir() / "oplog-import-export-build-ids.json"
+
+
class Build(NamedTuple):
name: str
bucket: str
id: str
-BUILDS = [
- Build("XB1Client", "fortnitegame.oplog.fortnite-main.xb1client", "09a75f7f3b7517653dcdaaa4"),
- Build("WindowsClient", "fortnitegame.oplog.fortnite-main.windowsclient", "09a75d977ef944ecfd0eddfd"),
- Build("SwitchClient", "fortnitegame.oplog.fortnite-main.switchclient", "09a74d03b3598ec94cfd2644"),
- Build("XSXClient", "fortnitegame.oplog.fortnite-main.xsxclient", "09a76c2bbd6cd78f4d40d9ea"),
- Build("Switch2Client", "fortnitegame.oplog.fortnite-main.switch2client", "09a7686b3d9faa78fb24a38f"),
- Build("PS4Client", "fortnitegame.oplog.fortnite-main.ps4client", "09a75b72d1c260ed26020140"),
- Build("LinuxServer", "fortnitegame.oplog.fortnite-main.linuxserver", "09a747f5e0ee83a04be013e6"),
- Build("IOSClient", "fortnitegame.oplog.fortnite-main.iosclient", "09a75f677e883325a209148c"),
- Build("Android_ASTCClient", "fortnitegame.oplog.fortnite-main.android_astcclient", "09a7422c08c6f37becc7d37f"),
-]
+def load_builds() -> tuple[str, list[Build]]:
+ if not _BUILD_IDS_PATH.exists():
+ print(f"Build IDs file not found: {_BUILD_IDS_PATH}")
+ answer = input("Run oplog-update-build-ids.py now to populate it? [y/N] ").strip().lower()
+ if answer == "y":
+ update_script = Path(__file__).parent / "oplog-update-build-ids.py"
+ subprocess.run([sys.executable, str(update_script)], check=True)
+ else:
+ sys.exit("Aborted. Run scripts/test_scripts/oplog-update-build-ids.py to populate it.")
+ with _BUILD_IDS_PATH.open() as f:
+ data: dict = json.load(f)
+ namespace = data.get("namespace", "")
+ if not namespace:
+ sys.exit(f"error: {_BUILD_IDS_PATH} is missing 'namespace'")
+ builds = []
+ for name, entry in data.get("builds", {}).items():
+ bucket = entry.get("bucket", "")
+ build_id = entry.get("buildId", "")
+ if not bucket or not build_id:
+ sys.exit(f"error: entry '{name}' in {_BUILD_IDS_PATH} is missing 'bucket' or 'buildId'")
+ builds.append(Build(name, bucket, build_id))
+ if not builds:
+ sys.exit(f"error: {_BUILD_IDS_PATH} contains no builds")
+ return namespace, builds
+
ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}")
@@ -50,6 +80,11 @@ SERVER_ARGS: tuple[str, ...] = (
)
+def zen_cmd(*args: str | Path, extra_zen_args: list[str] | None = None) -> list[str | Path]:
+ """Build a zen CLI command list, inserting extra_zen_args before subcommands."""
+ return [ZEN_EXE, *(extra_zen_args or []), *args]
+
+
def run(cmd: list[str | Path]) -> None:
try:
subprocess.run(cmd, check=True)
@@ -59,31 +94,33 @@ def run(cmd: list[str | Path]) -> None:
sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}")
-def stop_server(label: str, port: int) -> None:
+def stop_server(label: str, port: int, extra_zen_args: list[str] | None = None) -> None:
"""Stop a zen server. Tolerates failures so it is safe to call from finally blocks."""
print(f"--------- stopping {label}")
try:
- subprocess.run([ZEN_EXE, "down", "--port", str(port)])
+ subprocess.run(zen_cmd("down", "--port", str(port), extra_zen_args=extra_zen_args))
except OSError as e:
print(f"warning: could not stop {label}: {e}", file=sys.stderr)
print()
-def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None:
+def start_server(label: str, data_dir: Path, port: int, extra_zen_args: list[str] | None = None,
+ extra_server_args: list[str] | None = None) -> None:
print(f"--------- starting {label} {data_dir}")
- run([
- ZEN_EXE, "up", "--port", str(port), "--show-console", "--",
+ run(zen_cmd(
+ "up", "--port", str(port), "--show-console", "--",
f"--data-dir={data_dir}",
*SERVER_ARGS,
- *(extra_args or []),
- ])
+ *(extra_server_args or []),
+ extra_zen_args=extra_zen_args,
+ ))
print()
-def wipe_or_create(label: str, path: Path) -> None:
+def wipe_or_create(label: str, path: Path, extra_zen_args: list[str] | None = None) -> None:
if path.exists():
print(f"--------- cleaning {label} {path}")
- run([ZEN_EXE, "wipe", "-y", path])
+ run(zen_cmd("wipe", "-y", path, extra_zen_args=extra_zen_args))
else:
print(f"--------- creating {label} {path}")
path.mkdir(parents=True, exist_ok=True)
@@ -95,24 +132,39 @@ def check_prerequisites() -> None:
sys.exit(f"error: zen executable not found: {ZEN_EXE}")
-def setup_project(port: int) -> None:
+def setup_project(port: int, extra_zen_args: list[str] | None = None) -> None:
"""Create the FortniteGame project on the server at the given port."""
print("--------- creating FortniteGame project")
- run([ZEN_EXE, "project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update"])
+ run(zen_cmd("project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update",
+ extra_zen_args=extra_zen_args))
print()
-def setup_oplog(port: int, build_name: str) -> None:
+def setup_oplog(port: int, build_name: str, extra_zen_args: list[str] | None = None) -> None:
"""Create the oplog in the FortniteGame project on the server at the given port."""
print(f"--------- creating {build_name} oplog")
- run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build_name, "--force-update"])
+ run(zen_cmd("oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build_name, "--force-update",
+ extra_zen_args=extra_zen_args))
print()
def main() -> None:
global ZEN_EXE
- parser = argparse.ArgumentParser(description=__doc__)
+ # Split on '--' to separate script args from extra zen CLI args
+ script_argv: list[str] = []
+ extra_zen_args: list[str] = []
+ if "--" in sys.argv[1:]:
+ sep = sys.argv.index("--", 1)
+ script_argv = sys.argv[1:sep]
+ extra_zen_args = sys.argv[sep + 1:]
+ else:
+ script_argv = sys.argv[1:]
+
+ parser = argparse.ArgumentParser(
+ description=__doc__,
+ epilog="Any arguments after '--' are forwarded to every zen CLI invocation.",
+ )
parser.add_argument(
"positional_path",
nargs="?",
@@ -131,10 +183,10 @@ def main() -> None:
)
parser.add_argument(
"--data-path",
- default=Path(Path(__file__).stem + "_datadir"),
+ default=None,
type=Path,
metavar="PATH",
- help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)",
+ help="root path for all data directories",
)
parser.add_argument(
"--zen-exe-path",
@@ -143,15 +195,22 @@ def main() -> None:
metavar="PATH",
help=f"path to zen executable (default: {ZEN_EXE})",
)
- args = parser.parse_args()
+ args = parser.parse_args(script_argv)
data_path = args.positional_path
if data_path is None:
data_path = args.data_path
+ if data_path is None:
+ print("WARNING: This script may require up to 1TB of free disk space.")
+ raw = input("Enter root path for all data directories: ").strip()
+ if not raw:
+ sys.exit("error: data path is required")
+ data_path = Path(raw)
ZEN_EXE = args.zen_exe_positional
if ZEN_EXE is None:
ZEN_EXE = args.zen_exe_path
+ namespace, builds = load_builds()
zen_data_dir = data_path / "DDC" / "OplogsZen"
zen_cache_data_dir = data_path / "DDC" / "ZenBuildsCache"
zen_import_data_dir = data_path / "DDC" / "OplogsZenImport"
@@ -159,75 +218,81 @@ def main() -> None:
check_prerequisites()
- start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"])
+ start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT,
+ extra_zen_args=extra_zen_args, extra_server_args=["--buildstore-enabled"])
try:
- wipe_or_create("zenserver data", zen_data_dir)
- start_server("zenserver", zen_data_dir, ZEN_PORT)
+ wipe_or_create("zenserver data", zen_data_dir, extra_zen_args)
+ start_server("zenserver", zen_data_dir, ZEN_PORT, extra_zen_args=extra_zen_args)
try:
- setup_project(ZEN_PORT)
+ setup_project(ZEN_PORT, extra_zen_args)
- for build in BUILDS:
- setup_oplog(ZEN_PORT, build.name)
+ for build in builds:
+ setup_oplog(ZEN_PORT, build.name, extra_zen_args)
print(f"--------- importing {build.name} oplog")
- run([
- ZEN_EXE, "oplog-import",
+ run(zen_cmd(
+ "oplog-import",
f"--hosturl=127.0.0.1:{ZEN_PORT}",
"FortniteGame", build.name,
"--clean",
"--builds", "https://jupiter.devtools.epicgames.com",
- "--namespace", "fortnite.oplog",
+ "--namespace", namespace,
"--bucket", build.bucket,
"--builds-id", build.id,
f"--zen-cache-host={ZEN_CACHE}",
f"--zen-cache-upload={ZEN_CACHE_POPULATE}",
f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}",
- ])
+ extra_zen_args=extra_zen_args,
+ ))
print()
print(f"--------- validating {build.name} oplog")
- run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name])
+ run(zen_cmd("oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name,
+ extra_zen_args=extra_zen_args))
print()
- wipe_or_create("export folder", export_dir)
+ wipe_or_create("export folder", export_dir, extra_zen_args)
- for build in BUILDS:
+ for build in builds:
print(f"--------- exporting {build.name} oplog")
- run([
- ZEN_EXE, "oplog-export",
+ run(zen_cmd(
+ "oplog-export",
f"--hosturl=127.0.0.1:{ZEN_PORT}",
"FortniteGame", build.name,
"--file", export_dir,
"--forcetempblocks",
- ])
+ extra_zen_args=extra_zen_args,
+ ))
print()
finally:
- stop_server("zenserver", ZEN_PORT)
+ stop_server("zenserver", ZEN_PORT, extra_zen_args)
- wipe_or_create("alternate zenserver data", zen_import_data_dir)
- start_server("import zenserver", zen_import_data_dir, ZEN_PORT)
+ wipe_or_create("alternate zenserver data", zen_import_data_dir, extra_zen_args)
+ start_server("import zenserver", zen_import_data_dir, ZEN_PORT, extra_zen_args=extra_zen_args)
try:
- setup_project(ZEN_PORT)
+ setup_project(ZEN_PORT, extra_zen_args)
- for build in BUILDS:
- setup_oplog(ZEN_PORT, build.name)
+ for build in builds:
+ setup_oplog(ZEN_PORT, build.name, extra_zen_args)
print(f"--------- importing {build.name} oplog")
- run([
- ZEN_EXE, "oplog-import",
+ run(zen_cmd(
+ "oplog-import",
f"--hosturl=127.0.0.1:{ZEN_PORT}",
"FortniteGame", build.name,
"--file", export_dir,
- ])
+ extra_zen_args=extra_zen_args,
+ ))
print()
print(f"--------- validating {build.name} oplog")
- run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name])
+ run(zen_cmd("oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name,
+ extra_zen_args=extra_zen_args))
print()
finally:
- stop_server("alternative zenserver", ZEN_PORT)
+ stop_server("alternative zenserver", ZEN_PORT, extra_zen_args)
finally:
- stop_server("cache zenserver", ZEN_CACHE_PORT)
+ stop_server("cache zenserver", ZEN_CACHE_PORT, extra_zen_args)
if __name__ == "__main__":
diff --git a/scripts/test_scripts/oplog-update-build-ids.py b/scripts/test_scripts/oplog-update-build-ids.py
new file mode 100644
index 000000000..67e128c8e
--- /dev/null
+++ b/scripts/test_scripts/oplog-update-build-ids.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Update oplog-import-export-build-ids.json with build IDs at the highest common changelist across all buckets."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import platform
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux"
+_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower()
+_EXE_SUFFIX = ".exe" if sys.platform == "win32" else ""
+_DEFAULT_ZEN = Path(f"build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}")
+
+
+def _cache_dir() -> Path:
+ if sys.platform == "win32":
+ base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+ return base / "Temp" / "zen"
+ elif sys.platform == "darwin":
+ return Path.home() / "Library" / "Caches" / "zen"
+ else:
+ base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+ return base / "zen"
+
+
+_OUTPUT_PATH = _cache_dir() / "oplog-import-export-build-ids.json"
+
+# Maps build name -> Jupiter bucket
+_BUILDS: list[tuple[str, str]] = [
+ ("XB1Client", "fortnitegame.oplog.fortnite-main.xb1client"),
+ ("WindowsClient", "fortnitegame.oplog.fortnite-main.windowsclient"),
+ ("SwitchClient", "fortnitegame.oplog.fortnite-main.switchclient"),
+ ("XSXClient", "fortnitegame.oplog.fortnite-main.xsxclient"),
+ ("Switch2Client", "fortnitegame.oplog.fortnite-main.switch2client"),
+ ("PS4Client", "fortnitegame.oplog.fortnite-main.ps4client"),
+ ("PS5Client", "fortnitegame.oplog.fortnite-main.ps5client"),
+ ("LinuxServer", "fortnitegame.oplog.fortnite-main.linuxserver"),
+ ("IOSClient", "fortnitegame.oplog.fortnite-main.iosclient"),
+ ("Android_ASTCClient", "fortnitegame.oplog.fortnite-main.android_astcclient"),
+]
+
+
+def list_builds_for_bucket(zen: str, host: str, namespace: str, bucket: str) -> list[dict]:
+ """Run zen builds list for a single bucket and return the results array."""
+ with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp:
+ result_path = Path(tmp.name)
+
+ cmd = [
+ zen, "builds", "list",
+ "--namespace", namespace,
+ "--bucket", bucket,
+ "--host", host,
+ "--result-path", str(result_path),
+ ]
+
+ try:
+ subprocess.run(cmd, check=True, capture_output=True)
+ except FileNotFoundError:
+ sys.exit(f"error: zen binary not found: {zen}")
+ except subprocess.CalledProcessError as e:
+ sys.exit(
+ f"error: zen builds list failed for bucket '{bucket}' with exit code {e.returncode}\n"
+ f"stderr: {e.stderr.decode(errors='replace')}"
+ )
+
+ with result_path.open() as f:
+ data = json.load(f)
+ result_path.unlink(missing_ok=True)
+
+ return data.get("results", [])
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(
+ description="Refresh oplog-import-export-build-ids.json with build IDs at the highest changelist present in all buckets."
+ )
+ parser.add_argument("--host", default="https://jupiter.devtools.epicgames.com", help="Jupiter host URL")
+ parser.add_argument("--zen", default=str(_DEFAULT_ZEN), help="Path to the zen binary")
+ parser.add_argument("--namespace", default="fortnite.oplog", help="Builds storage namespace")
+ args = parser.parse_args()
+
+ # For each bucket, fetch results and build a changelist -> buildId map.
+ # bucket_cl_map[bucket] = { changelist_int: buildId_str, ... }
+ bucket_cl_map: dict[str, dict[int, str]] = {}
+
+ for name, bucket in _BUILDS:
+ print(f"Querying {name} ({bucket}) ...")
+ results = list_builds_for_bucket(args.zen, args.host, args.namespace, bucket)
+ if not results:
+ sys.exit(f"error: no results for bucket '{bucket}' (build '{name}')")
+
+ cl_map: dict[int, str] = {}
+ for entry in results:
+ build_id = entry.get("buildId", "")
+ metadata = entry.get("metadata") or {}
+ cl = metadata.get("changelist")
+ if build_id and cl is not None:
+ # Keep first occurrence (most recent) per changelist
+ if cl not in cl_map:
+ cl_map[int(cl)] = build_id
+
+ if not cl_map:
+ sys.exit(
+ f"error: bucket '{bucket}' (build '{name}') returned {len(results)} entries "
+ "but none had both buildId and changelist in metadata"
+ )
+
+ print(f" {len(cl_map)} distinct changelists, latest CL {max(cl_map)}")
+ bucket_cl_map[bucket] = cl_map
+
+ # Find the highest changelist present in every bucket's result set.
+ common_cls = set(next(iter(bucket_cl_map.values())).keys())
+ for bucket, cl_map in bucket_cl_map.items():
+ common_cls &= set(cl_map.keys())
+
+ if not common_cls:
+ sys.exit(
+ "error: no changelist is present in all buckets.\n"
+ "Per-bucket CL ranges:\n"
+ + "\n".join(
+ f" {name} ({bucket}): {min(bucket_cl_map[bucket])} – {max(bucket_cl_map[bucket])}"
+ for name, bucket in _BUILDS
+ )
+ )
+
+ best_cl = max(common_cls)
+ print(f"\nHighest common changelist: {best_cl}")
+
+ build_ids: dict[str, dict[str, str]] = {}
+ for name, bucket in _BUILDS:
+ build_id = bucket_cl_map[bucket][best_cl]
+ build_ids[name] = {"bucket": bucket, "buildId": build_id}
+ print(f" {name}: {build_id}")
+
+ output = {"namespace": args.namespace, "builds": build_ids}
+ _OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+ with _OUTPUT_PATH.open("w") as f:
+ json.dump(output, f, indent=2)
+ f.write("\n")
+
+ print(f"\nWrote {_OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/updatefrontend.lua b/scripts/updatefrontend.lua
deleted file mode 100644
index ab37819d7..000000000
--- a/scripts/updatefrontend.lua
+++ /dev/null
@@ -1,111 +0,0 @@
--- Copyright Epic Games, Inc. All Rights Reserved.
-
---------------------------------------------------------------------------------
-local function _exec(cmd, ...)
- local args = {}
- for _, arg in pairs({...}) do
- if arg then
- table.insert(args, arg)
- end
- end
-
- print("--", cmd, table.unpack(args))
- local ret = os.execv(cmd, args)
- print()
- return ret
-end
-
---------------------------------------------------------------------------------
-local function _zip(store_only, zip_path, ...)
- -- Here's the rules; if len(...) is 1 and it is a dir then create a zip with
- -- archive paths like this;
- --
- -- glob(foo/bar/**) -> foo/bar/abc, foo/bar/dir/123 -> zip(abc, dir/123)
- --
- -- Otherwise assume ... is file paths and add without leading directories;
- --
- -- foo/abc, bar/123 -> zip(abc, 123)
-
- zip_path = path.absolute(zip_path)
- os.tryrm(zip_path)
-
- local inputs = {...}
-
- local source_dir = nil
- if #inputs == 1 and os.isdir(inputs[1]) then
- source_dir = inputs[1]
- end
-
- import("detect.tools.find_7z")
- local cmd_7z = find_7z()
- if cmd_7z then
- input_paths = {}
- if source_dir then
- -- Suffixing a directory path with a "/." will have 7z set the path
- -- for archived files relative to that directory.
- input_paths = { path.join(source_dir, ".") }
- else
- for _, input_path in pairs(inputs) do
- -- If there is a "/./" anywhere in file paths then 7z drops all
- -- directory information and just archives the file by name
- input_path = path.relative(input_path, ".")
- if input_path:sub(2,2) ~= ":" then
- input_path = "./"..input_path
- end
- table.insert(input_paths, input_path)
- end
- end
-
- compression_level = "-mx1"
- if store_only then
- compression_level = "-mx0"
- end
-
- local ret = _exec(cmd_7z, "a", compression_level, zip_path, table.unpack(input_paths))
- if ret > 0 then
- raise("Received error from 7z")
- end
- return
- end
-
- print("7z not found, falling back to zip")
-
- import("detect.tools.find_zip")
- zip_cmd = find_zip()
- if zip_cmd then
- local input_paths = inputs
- local cwd = os.curdir()
- if source_dir then
- os.cd(source_dir)
- input_paths = { "." }
- end
-
- compression_level = "-1"
- if store_only then
- compression_level = "-0"
- end
-
- local strip_leading_path = nil
- if not source_dir then
- strip_leading_path = "--junk-paths"
- end
-
- local ret = _exec(zip_cmd, "-r", compression_level, strip_leading_path, zip_path, table.unpack(input_paths))
- if ret > 0 then
- raise("Received error from zip")
- end
-
- os.cd(cwd)
- return
- end
- print("zip not found")
-
- raise("Unable to find a suitable zip tool")
-end
-
---------------------------------------------------------------------------------
-function main()
- local zip_path = "src/zenserver/frontend/html.zip"
- local content_dir = "src/zenserver/frontend/html/"
- _zip(true, zip_path, content_dir)
-end
diff --git a/src/transports/winsock/winsock.cpp b/src/transports/winsock/winsock.cpp
index f98984726..c1f4f6abe 100644
--- a/src/transports/winsock/winsock.cpp
+++ b/src/transports/winsock/winsock.cpp
@@ -271,7 +271,7 @@ WinsockTransportPlugin::Initialize(TransportServer* ServerInterface)
m_ServerInterface = ServerInterface;
m_ListenSocket = socket(AF_INET6, SOCK_STREAM, 0);
- if (m_ListenSocket == SOCKET_ERROR || m_ListenSocket == INVALID_SOCKET)
+ if (m_ListenSocket == INVALID_SOCKET)
{
throw std::system_error(std::error_code(WSAGetLastError(), std::system_category()),
"socket creation failed in HTTP plugin server init");
@@ -302,7 +302,7 @@ WinsockTransportPlugin::Initialize(TransportServer* ServerInterface)
do
{
- if (SOCKET ClientSocket = accept(m_ListenSocket, NULL, NULL); ClientSocket != SOCKET_ERROR)
+ if (SOCKET ClientSocket = accept(m_ListenSocket, NULL, NULL); ClientSocket != INVALID_SOCKET)
{
int Flag = 1;
setsockopt(ClientSocket, IPPROTO_TCP, TCP_NODELAY, (char*)&Flag, sizeof(Flag));
diff --git a/src/zencore/include/zencore/logbase.h b/src/zencore/include/zencore/logbase.h
index ad2ab218d..046e96db3 100644
--- a/src/zencore/include/zencore/logbase.h
+++ b/src/zencore/include/zencore/logbase.h
@@ -101,7 +101,7 @@ struct LoggerRef
inline logging::Logger* operator->() const;
inline logging::Logger& operator*() const;
- bool ShouldLog(logging::LogLevel Level) const { return m_Logger->ShouldLog(Level); }
+ bool ShouldLog(logging::LogLevel Level) const { return m_Logger && m_Logger->ShouldLog(Level); }
void SetLogLevel(logging::LogLevel NewLogLevel) { m_Logger->SetLevel(NewLogLevel); }
logging::LogLevel GetLogLevel() { return m_Logger->GetLevel(); }
diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h
index 5a8a66fae..7b46f0e38 100644
--- a/src/zencore/include/zencore/string.h
+++ b/src/zencore/include/zencore/string.h
@@ -331,7 +331,7 @@ public:
return AppendAscii(Str);
}
-#if defined(__clang__) && !defined(__APPLE__)
+#if defined(__clang__) && !defined(__APPLE__) && !defined(_MSC_VER)
/* UE Toolchain Clang has different types for int64_t and long long so an override is
needed here. Without it, Clang can't disambiguate integer overloads */
inline StringBuilderImpl& operator<<(long long n)
diff --git a/src/zencore/memtrack/callstacktrace.cpp b/src/zencore/memtrack/callstacktrace.cpp
index 4a7068568..013c51535 100644
--- a/src/zencore/memtrack/callstacktrace.cpp
+++ b/src/zencore/memtrack/callstacktrace.cpp
@@ -912,92 +912,100 @@ FBacktracer::GetBacktraceId(void* AddressOfReturnAddress)
# else // UE_CALLSTACK_TRACE_USE_UNWIND_TABLES
-namespace zen {
+////////////////////////////////////////////////////////////////////////////////
+class FBacktracer
+{
+public:
+ FBacktracer(FMalloc* InMalloc);
+ ~FBacktracer();
+ static FBacktracer* Get();
+ inline uint32_t GetBacktraceId(void* AddressOfReturnAddress);
+ uint32_t GetBacktraceId(uint64_t ReturnAddress);
+ void AddModule(uintptr_t Base, const char16_t* Name) {}
+ void RemoveModule(uintptr_t Base) {}
- ////////////////////////////////////////////////////////////////////////////////
- class FBacktracer
- {
- public:
- FBacktracer(FMalloc* InMalloc);
- ~FBacktracer();
- static FBacktracer* Get();
- inline uint32_t GetBacktraceId(void* AddressOfReturnAddress);
- uint32_t GetBacktraceId(uint64_t ReturnAddress);
- void AddModule(uintptr_t Base, const char16_t* Name) {}
- void RemoveModule(uintptr_t Base) {}
-
- private:
- static FBacktracer* Instance;
- FMalloc* Malloc;
- FCallstackTracer CallstackTracer;
- };
+private:
+ static FBacktracer* Instance;
+ FMalloc* Malloc;
+ FCallstackTracer CallstackTracer;
+};
- ////////////////////////////////////////////////////////////////////////////////
- FBacktracer* FBacktracer::Instance = nullptr;
+////////////////////////////////////////////////////////////////////////////////
+FBacktracer* FBacktracer::Instance = nullptr;
- ////////////////////////////////////////////////////////////////////////////////
- FBacktracer::FBacktracer(FMalloc* InMalloc) : Malloc(InMalloc), CallstackTracer(InMalloc) { Instance = this; }
+////////////////////////////////////////////////////////////////////////////////
+FBacktracer::FBacktracer(FMalloc* InMalloc) : Malloc(InMalloc), CallstackTracer(InMalloc)
+{
+ Instance = this;
+}
- ////////////////////////////////////////////////////////////////////////////////
- FBacktracer::~FBacktracer() {}
+////////////////////////////////////////////////////////////////////////////////
+FBacktracer::~FBacktracer()
+{
+}
- ////////////////////////////////////////////////////////////////////////////////
- FBacktracer* FBacktracer::Get() { return Instance; }
+////////////////////////////////////////////////////////////////////////////////
+FBacktracer*
+FBacktracer::Get()
+{
+ return Instance;
+}
- ////////////////////////////////////////////////////////////////////////////////
- uint32_t FBacktracer::GetBacktraceId(void* AddressOfReturnAddress)
- {
- const uint64_t ReturnAddress = *(uint64_t*)AddressOfReturnAddress;
- return GetBacktraceId(ReturnAddress);
- }
+////////////////////////////////////////////////////////////////////////////////
+uint32_t
+FBacktracer::GetBacktraceId(void* AddressOfReturnAddress)
+{
+ const uint64_t ReturnAddress = *(uint64_t*)AddressOfReturnAddress;
+ return GetBacktraceId(ReturnAddress);
+}
- ////////////////////////////////////////////////////////////////////////////////
- uint32_t FBacktracer::GetBacktraceId(uint64_t ReturnAddress)
- {
+////////////////////////////////////////////////////////////////////////////////
+uint32_t
+FBacktracer::GetBacktraceId(uint64_t ReturnAddress)
+{
# if !UE_BUILD_SHIPPING
- uint64_t StackFrames[256];
- int32_t NumStackFrames = FPlatformStackWalk::CaptureStackBackTrace(StackFrames, UE_ARRAY_COUNT(StackFrames));
- if (NumStackFrames > 0)
+ uint64_t StackFrames[256];
+ int32_t NumStackFrames = FPlatformStackWalk::CaptureStackBackTrace(StackFrames, UE_ARRAY_COUNT(StackFrames));
+ if (NumStackFrames > 0)
+ {
+ FCallstackTracer::FBacktraceEntry BacktraceEntry;
+ uint64_t BacktraceId = 0;
+ uint32_t FrameIdx = 0;
+ bool bUseAddress = false;
+ for (int32_t Index = 0; Index < NumStackFrames; Index++)
{
- FCallstackTracer::FBacktraceEntry BacktraceEntry;
- uint64_t BacktraceId = 0;
- uint32_t FrameIdx = 0;
- bool bUseAddress = false;
- for (int32_t Index = 0; Index < NumStackFrames; Index++)
+ if (!bUseAddress)
{
- if (!bUseAddress)
+ // start using backtrace only after ReturnAddress
+ if (StackFrames[Index] == (uint64_t)ReturnAddress)
{
- // start using backtrace only after ReturnAddress
- if (StackFrames[Index] == (uint64_t)ReturnAddress)
- {
- bUseAddress = true;
- }
- }
- if (bUseAddress || NumStackFrames == 1)
- {
- uint64_t RetAddr = StackFrames[Index];
- StackFrames[FrameIdx++] = RetAddr;
-
- // This is a simple order-dependent LCG. Should be sufficient enough
- BacktraceId += RetAddr;
- BacktraceId *= 0x30be8efa499c249dull;
+ bUseAddress = true;
}
}
+ if (bUseAddress || NumStackFrames == 1)
+ {
+ uint64_t RetAddr = StackFrames[Index];
+ StackFrames[FrameIdx++] = RetAddr;
- // Save the collected id
- BacktraceEntry.Hash = BacktraceId;
- BacktraceEntry.FrameCount = FrameIdx;
- BacktraceEntry.Frames = StackFrames;
-
- // Add to queue to be processed. This might block until there is room in the
- // queue (i.e. the processing thread has caught up processing).
- return CallstackTracer.AddCallstack(BacktraceEntry);
+ // This is a simple order-dependent LCG. Should be sufficient enough
+ BacktraceId += RetAddr;
+ BacktraceId *= 0x30be8efa499c249dull;
+ }
}
-# endif
- return 0;
+ // Save the collected id
+ BacktraceEntry.Hash = BacktraceId;
+ BacktraceEntry.FrameCount = FrameIdx;
+ BacktraceEntry.Frames = StackFrames;
+
+ // Add to queue to be processed. This might block until there is room in the
+ // queue (i.e. the processing thread has caught up processing).
+ return CallstackTracer.AddCallstack(BacktraceEntry);
}
+# endif
+ return 0;
+}
}
# endif // UE_CALLSTACK_TRACE_USE_UNWIND_TABLES
@@ -1047,7 +1055,7 @@ CallstackTrace_GetCurrentId()
# if PLATFORM_USE_CALLSTACK_ADDRESS_POINTER
return Instance->GetBacktraceId(StackAddress);
# else
- return Instance->GetBacktraceId((uint64_t)StackAddress);
+ return Instance->GetBacktraceId((uint64_t)StackAddress);
# endif
}
diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index deeeb6c85..9f49802a0 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -36,15 +36,17 @@
namespace zen {
+#if ZEN_WITH_CPR
extern HttpClientBase* CreateCprHttpClient(std::string_view BaseUri,
const HttpClientSettings& ConnectionSettings,
std::function<bool()>&& CheckIfAbortFunction);
+#endif
extern HttpClientBase* CreateCurlHttpClient(std::string_view BaseUri,
const HttpClientSettings& ConnectionSettings,
std::function<bool()>&& CheckIfAbortFunction);
-static HttpClientBackend g_DefaultHttpClientBackend = HttpClientBackend::kCpr;
+static HttpClientBackend g_DefaultHttpClientBackend = HttpClientBackend::kCurl;
void
SetDefaultHttpClientBackend(HttpClientBackend Backend)
@@ -55,11 +57,14 @@ SetDefaultHttpClientBackend(HttpClientBackend Backend)
void
SetDefaultHttpClientBackend(std::string_view Backend)
{
+#if ZEN_WITH_CPR
if (Backend == "cpr")
{
g_DefaultHttpClientBackend = HttpClientBackend::kCpr;
}
- else if (Backend == "curl")
+ else
+#endif
+ if (Backend == "curl")
{
g_DefaultHttpClientBackend = HttpClientBackend::kCurl;
}
@@ -363,13 +368,15 @@ HttpClient::HttpClient(std::string_view BaseUri, const HttpClientSettings& Conne
switch (EffectiveBackend)
{
- case HttpClientBackend::kCurl:
- m_Inner = CreateCurlHttpClient(BaseUri, ConnectionSettings, std::move(CheckIfAbortFunction));
- break;
+#if ZEN_WITH_CPR
case HttpClientBackend::kCpr:
- default:
m_Inner = CreateCprHttpClient(BaseUri, ConnectionSettings, std::move(CheckIfAbortFunction));
break;
+#endif
+ case HttpClientBackend::kCurl:
+ default:
+ m_Inner = CreateCurlHttpClient(BaseUri, ConnectionSettings, std::move(CheckIfAbortFunction));
+ break;
}
}
diff --git a/src/zenhttp/include/zenhttp/cprutils.h b/src/zenhttp/include/zenhttp/cprutils.h
index c252a5d99..3cfe652c5 100644
--- a/src/zenhttp/include/zenhttp/cprutils.h
+++ b/src/zenhttp/include/zenhttp/cprutils.h
@@ -2,17 +2,19 @@
#pragma once
-#include <zencore/compactbinary.h>
-#include <zencore/compactbinaryvalidation.h>
-#include <zencore/iobuffer.h>
-#include <zencore/string.h>
-#include <zenhttp/formatters.h>
-#include <zenhttp/httpclient.h>
-#include <zenhttp/httpcommon.h>
+#if ZEN_WITH_CPR
+
+# include <zencore/compactbinary.h>
+# include <zencore/compactbinaryvalidation.h>
+# include <zencore/iobuffer.h>
+# include <zencore/string.h>
+# include <zenhttp/formatters.h>
+# include <zenhttp/httpclient.h>
+# include <zenhttp/httpcommon.h>
ZEN_THIRD_PARTY_INCLUDES_START
-#include <cpr/response.h>
-#include <fmt/format.h>
+# include <cpr/response.h>
+# include <fmt/format.h>
ZEN_THIRD_PARTY_INCLUDES_END
template<>
@@ -92,3 +94,5 @@ struct fmt::formatter<cpr::Response>
}
}
};
+
+#endif // ZEN_WITH_CPR
diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h
index e95d78772..e878c900f 100644
--- a/src/zenhttp/include/zenhttp/httpclient.h
+++ b/src/zenhttp/include/zenhttp/httpclient.h
@@ -52,7 +52,9 @@ enum class HttpClientErrorCode : int
enum class HttpClientBackend : uint8_t
{
kDefault,
+#if ZEN_WITH_CPR
kCpr,
+#endif
kCurl,
};
diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp
index 4bf8c61bb..a1bb719c8 100644
--- a/src/zenhttp/servers/httpplugin.cpp
+++ b/src/zenhttp/servers/httpplugin.cpp
@@ -147,7 +147,7 @@ public:
HttpPluginServerRequest& operator=(const HttpPluginServerRequest&) = delete;
// As this is plugin transport connection used for specialized connections we assume it is not a machine local connection
- virtual bool IsLocalMachineRequest() const /* override*/ { return false; }
+ bool IsLocalMachineRequest() const override { return false; }
virtual std::string_view GetAuthorizationHeader() const override;
virtual Oid ParseSessionId() const override;
virtual uint32_t ParseRequestId() const override;
diff --git a/src/zenhttp/xmake.lua b/src/zenhttp/xmake.lua
index 9b461662e..b4c65ea96 100644
--- a/src/zenhttp/xmake.lua
+++ b/src/zenhttp/xmake.lua
@@ -8,7 +8,12 @@ target('zenhttp')
add_files("servers/httpsys.cpp", {unity_ignored=true})
add_files("servers/wshttpsys.cpp", {unity_ignored=true})
add_includedirs("include", {public=true})
- add_deps("zencore", "zentelemetry", "transport-sdk", "asio", "cpr")
+ add_deps("zencore", "zentelemetry", "transport-sdk", "asio")
+ if has_config("zencpr") then
+ add_deps("cpr")
+ else
+ remove_files("clients/httpclientcpr.cpp")
+ end
add_packages("http_parser", "json11")
add_options("httpsys")
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
deleted file mode 100644
index 58778a592..000000000
--- a/src/zenserver/frontend/html.zip
+++ /dev/null
Binary files differ
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index 3cfaa956d..52889fff5 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -12,12 +12,14 @@ target("zenserver")
"zenremotestore",
"zenstore",
"zentelemetry",
- "zenutil",
- "zenvfs")
+ "zenutil")
+ if is_plat("windows") then
+ add_deps("zenvfs")
+ end
add_headerfiles("**.h")
add_rules("utils.bin2c", {extensions = {".zip"}})
add_files("**.cpp")
- add_files("frontend/*.zip")
+ add_files("frontend/html.zip")
add_files("zenserver.cpp", {unity_ignored = true })
if is_plat("linux") and not (get_config("toolchain") or ""):find("clang") then
@@ -78,7 +80,45 @@ target("zenserver")
add_ldflags("-framework SystemConfiguration")
end
- -- to work around some unfortunate Ctrl-C behaviour on Linux/Mac due to
+ on_load(function(target)
+ local html_dir = path.join(os.projectdir(), "src/zenserver/frontend/html")
+ local zip_path = path.join(os.projectdir(), "src/zenserver/frontend/html.zip")
+
+ -- Check if zip needs regeneration
+ local need_update = not os.isfile(zip_path)
+ if not need_update then
+ local zip_mtime = os.mtime(zip_path)
+ for _, file in ipairs(os.files(path.join(html_dir, "**"))) do
+ if os.mtime(file) > zip_mtime then
+ need_update = true
+ break
+ end
+ end
+ end
+
+ if need_update then
+ print("Regenerating frontend zip...")
+ os.tryrm(zip_path)
+
+ import("detect.tools.find_7z")
+ local cmd_7z = find_7z()
+ if cmd_7z then
+ os.execv(cmd_7z, {"a", "-mx0", zip_path, path.join(html_dir, ".")})
+ else
+ import("detect.tools.find_zip")
+ local zip_cmd = find_zip()
+ if zip_cmd then
+ local oldir = os.cd(html_dir)
+ os.execv(zip_cmd, {"-r", "-0", zip_path, "."})
+ os.cd(oldir)
+ else
+ raise("Unable to find a suitable zip tool (need 7z or zip)")
+ end
+ end
+ end
+ end)
+
+ -- to work around some unfortunate Ctrl-C behaviour on Linux/Mac due to
-- our use of setsid() at startup we pass in `--no-detach` to zenserver
-- ensure that it recieves signals when the user requests termination
on_run(function(target)
diff --git a/src/zenstore/xmake.lua b/src/zenstore/xmake.lua
index ea8155e94..94c2b51ca 100644
--- a/src/zenstore/xmake.lua
+++ b/src/zenstore/xmake.lua
@@ -6,6 +6,11 @@ target('zenstore')
add_headerfiles("**.h")
add_files("**.cpp")
add_includedirs("include", {public=true})
- add_deps("zencore", "zentelemetry", "zenutil", "zenvfs")
+ add_deps("zencore", "zentelemetry", "zenutil")
+ if is_plat("windows") then
+ add_deps("zenvfs")
+ else
+ add_includedirs("$(projectdir)/src/zenvfs/include", {public=true})
+ end
add_deps("robin-map")
add_packages("eastl", {public=true});
diff --git a/src/zenutil/workerpools.cpp b/src/zenutil/workerpools.cpp
index 1bab39b2a..25f961f77 100644
--- a/src/zenutil/workerpools.cpp
+++ b/src/zenutil/workerpools.cpp
@@ -25,9 +25,9 @@ namespace {
struct WorkerPool
{
- std::unique_ptr<WorkerThreadPool> Pool;
const int TreadCount;
const std::string_view Name;
+ std::unique_ptr<WorkerThreadPool> Pool;
};
WorkerPool BurstLargeWorkerPool = {.TreadCount = LargeWorkerThreadPoolTreadCount, .Name = "large"};
diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua
index 3e8d88c5d..013ed8de3 100644
--- a/thirdparty/xmake.lua
+++ b/thirdparty/xmake.lua
@@ -37,7 +37,7 @@ target('rpmalloc')
set_group('thirdparty')
set_languages("c17", "cxx20")
if is_os("windows") then
- add_cflags("/experimental:c11atomics", {force=true})
+ add_cflags("/experimental:c11atomics", {force=true, tools="cl"})
end
add_defines("RPMALLOC_FIRST_CLASS_HEAPS=1", "ENABLE_STATISTICS=1", "ENABLE_OVERRIDE=0")
add_files("rpmalloc/rpmalloc.c")
@@ -84,7 +84,7 @@ target("blake3")
add_includedirs("blake3/c", {public=true})
if is_os("windows") then
- add_cflags("/experimental:c11atomics")
+ add_cflags("/experimental:c11atomics", {tools="cl"})
add_cflags("/wd4245", {force = true}) -- conversion from 'type1' to 'type2', possible loss of data
elseif is_os("macosx") then
add_cflags("-Wno-unused-function")
diff --git a/xmake.lua b/xmake.lua
index f7f0c8df3..0d68e136c 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -191,6 +191,30 @@ if is_os("linux") or is_os("macosx") then
add_cxxflags("-Wno-unused-but-set-variable", {tools="gcc"})
end
+if get_config("toolchain") == "clang-cl" then
+ add_cxxflags("-Wno-unknown-warning-option", {force=true})
+ add_cxxflags("-Wno-cast-function-type-mismatch", {force=true})
+ add_cxxflags("-Wno-delete-non-abstract-non-virtual-dtor", {force=true})
+ add_cxxflags("-Wno-format", {force=true})
+ add_cxxflags("-Wno-implicit-fallthrough", {force=true})
+ add_cxxflags("-Wno-inconsistent-missing-override", {force=true})
+ add_cxxflags("-Wno-missing-field-initializers", {force=true})
+ add_cxxflags("-Wno-parentheses-equality", {force=true})
+ add_cxxflags("-Wno-reorder-ctor", {force=true})
+ add_cxxflags("-Wno-sign-compare", {force=true})
+ add_cxxflags("-Wno-strict-aliasing", {force=true})
+ add_cxxflags("-Wno-switch", {force=true})
+ add_cxxflags("-Wno-unused-but-set-variable", {force=true})
+ add_cxxflags("-Wno-unused-lambda-capture", {force=true})
+ add_cxxflags("-Wno-unused-parameter", {force=true})
+ add_cxxflags("-Wno-unused-private-field", {force=true})
+ add_cxxflags("-Wno-unused-value", {force=true})
+ add_cxxflags("-Wno-unused-variable", {force=true})
+ add_cxxflags("-Wno-vla-cxx-extension", {force=true})
+ add_cflags("-Wno-unknown-warning-option", {force=true})
+ add_cflags("-Wno-unused-command-line-argument", {force=true})
+end
+
if is_os("macosx") then
-- silence warnings about -Wno-vla-cxx-extension since to my knowledge we can't
-- detect the clang version used in Xcode and only recent versions contain this flag
@@ -288,6 +312,13 @@ option("zentrace")
option_end()
add_define_by_config("ZEN_WITH_TRACE", "zentrace")
+option("zencpr")
+ set_default(true)
+ set_showmenu(true)
+ set_description("Enable CPR HTTP client backend")
+option_end()
+add_define_by_config("ZEN_WITH_CPR", "zencpr")
+
set_warnings("allextra", "error")
set_languages("cxx20")
@@ -330,7 +361,9 @@ end
includes("src/zenstore", "src/zenstore-test")
includes("src/zentelemetry", "src/zentelemetry-test")
includes("src/zenutil", "src/zenutil-test")
-includes("src/zenvfs")
+if is_plat("windows") then
+ includes("src/zenvfs")
+end
includes("src/zenserver", "src/zenserver-test")
includes("src/zen")
includes("src/zentest-appstub")
@@ -368,16 +401,6 @@ task("kill")
end
end)
-task("updatefrontend")
- set_menu {
- usage = "xmake updatefrontend",
- description = "Create Zip of the frontend/html folder for bundling with zenserver executable",
- }
- on_run(function()
- import("scripts.updatefrontend")
- updatefrontend()
- end)
-
task("precommit")
set_menu {
usage = "xmake precommit",