Merge branch 'main' into sb/compute-schedulersb/compute-scheduler

1 files changed, 56 insertions, 18 deletions

diff --git a/scripts/test_scripts/builds-download-upload-test.py b/scripts/test_scripts/builds-download-upload-test.py
index e4fee7cb8..8ff5245c1 100644
--- a/scripts/test_scripts/builds-download-upload-test.py
+++ b/scripts/test_scripts/builds-download-upload-test.py
@@ -4,6 +4,8 @@
 from __future__ import annotations
 
 import argparse
+import json
+import os
 import platform
 import subprocess
 import sys
@@ -15,22 +17,51 @@ _ARCH = "x64" if sys.platform == "win32" else platform.machine().lower()
 _EXE_SUFFIX = ".exe" if sys.platform == "win32" else ""
 
 
+def _cache_dir() -> Path:
+    if sys.platform == "win32":
+        base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+        return base / "Temp" / "zen"
+    elif sys.platform == "darwin":
+        return Path.home() / "Library" / "Caches" / "zen"
+    else:
+        base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+        return base / "zen"
+
+
+_BUILD_IDS_PATH = _cache_dir() / "builds-download-upload-build-ids.json"
+
+
 class Build(NamedTuple):
     name: str
     bucket: str
     id: str
 
 
-BUILDS = [
-    Build("XB1Client",     "fortnitegame.staged-build.fortnite-main.xb1-client",     "09a7616c1a388dfe6056aa57"),
-    Build("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client", "09a762c81e2cf213142d0ce5"),
-    Build("SwitchClient",  "fortnitegame.staged-build.fortnite-main.switch-client",  "09a75bf9c3ce75bce09f644f"),
-    Build("LinuxServer",   "fortnitegame.staged-build.fortnite-main.linux-server",   "09a750ac155eb3e3b62e87e0"),
-    Build("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client", "09a78f3df07b289691ec5710"),
-    Build("PS4Client",     "fortnitegame.staged-build.fortnite-main.ps4-client",     "09a76ea92ad301d4724fafad"),
-    Build("IOSClient",     "fortnitegame.staged-build.fortnite-main.ios-client",     "09a7816fa26c23362fef0c5d"),
-    Build("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client", "09a76725f1620d62c6be06e4"),
-]
+def load_builds() -> tuple[str, list[Build]]:
+    if not _BUILD_IDS_PATH.exists():
+        print(f"Build IDs file not found: {_BUILD_IDS_PATH}")
+        answer = input("Run builds-download-upload-update-build-ids.py now to populate it? [y/N] ").strip().lower()
+        if answer == "y":
+            update_script = Path(__file__).parent / "builds-download-upload-update-build-ids.py"
+            subprocess.run([sys.executable, str(update_script)], check=True)
+        else:
+            sys.exit("Aborted. Run scripts/test_scripts/builds-download-upload-update-build-ids.py to populate it.")
+    with _BUILD_IDS_PATH.open() as f:
+        data: dict = json.load(f)
+    namespace = data.get("namespace", "")
+    if not namespace:
+        sys.exit(f"error: {_BUILD_IDS_PATH} is missing 'namespace'")
+    builds = []
+    for name, entry in data.get("builds", {}).items():
+        bucket = entry.get("bucket", "")
+        build_id = entry.get("buildId", "")
+        if not bucket or not build_id:
+            sys.exit(f"error: entry '{name}' in {_BUILD_IDS_PATH} is missing 'bucket' or 'buildId'")
+        builds.append(Build(name, bucket, build_id))
+    if not builds:
+        sys.exit(f"error: {_BUILD_IDS_PATH} contains no builds")
+    return namespace, builds
+
 
 ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}")
 ZEN_METADATA_DIR: Path = Path(__file__).resolve().parent / "metadatas"
@@ -99,12 +130,12 @@ def wipe_or_create(label: str, path: Path, extra_zen_args: list[str] | None = No
     print()
 
 
-def check_prerequisites() -> None:
+def check_prerequisites(builds: list[Build]) -> None:
     if not ZEN_EXE.is_file():
         sys.exit(f"error: zen executable not found: {ZEN_EXE}")
     if not ZEN_METADATA_DIR.is_dir():
         sys.exit(f"error: metadata directory not found: {ZEN_METADATA_DIR}")
-    for build in BUILDS:
+    for build in builds:
         metadata = ZEN_METADATA_DIR / f"{build.name}.json"
         if not metadata.is_file():
             sys.exit(f"error: metadata file not found: {metadata}")
@@ -145,10 +176,10 @@ def main() -> None:
     )
     parser.add_argument(
         "--data-path",
-        default=Path(Path(__file__).stem + "_datadir"),
+        default=None,
         type=Path,
         metavar="PATH",
-        help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)",
+        help="root path for all data directories",
     )
     parser.add_argument(
         "--zen-exe-path",
@@ -162,17 +193,24 @@ def main() -> None:
     data_path = args.positional_path
     if data_path is None:
         data_path = args.data_path
+    if data_path is None:
+        print("WARNING: This script may require up to 1TB of free disk space.")
+        raw = input("Enter root path for all data directories: ").strip()
+        if not raw:
+            sys.exit("error: data path is required")
+        data_path = Path(raw)
 
     ZEN_EXE = args.zen_exe_positional
     if ZEN_EXE is None:
         ZEN_EXE = args.zen_exe_path
+    namespace, builds = load_builds()
     zen_system_dir = data_path / "system"
     zen_download_dir = data_path / "Download"
     zen_cache_data_dir = data_path / "ZenBuildsCache"
     zen_upload_dir = data_path / "Upload"
     zen_chunk_cache_path = data_path / "ChunkCache"
 
-    check_prerequisites()
+    check_prerequisites(builds)
 
     start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT,
                  extra_zen_args=extra_zen_args, extra_server_args=["--buildstore-enabled"])
@@ -180,12 +218,12 @@ def main() -> None:
         wipe_or_create("download folder", zen_download_dir, extra_zen_args)
         wipe_or_create("system folder", zen_system_dir, extra_zen_args)
 
-        for build in BUILDS:
+        for build in builds:
             print(f"--------- importing {build.name} build")
             run(zen_cmd(
                 "builds", "download",
                 "--host", "https://jupiter.devtools.epicgames.com",
-                "--namespace", "fortnite.oplog",
+                "--namespace", namespace,
                 "--bucket", build.bucket,
                 "--build-id", build.id,
                 "--local-path", zen_download_dir / build.name,
@@ -199,7 +237,7 @@ def main() -> None:
 
         wipe_or_create("upload folder", zen_upload_dir, extra_zen_args)
 
-        for build in BUILDS:
+        for build in builds:
             print(f"--------- exporting {build.name} build")
             run(zen_cmd(
                 "builds", "upload",