Merged main.

2 files changed, 284 insertions, 4 deletions

diff --git a/scripts/formatcode.py b/scripts/formatcode.py
index 423d2b4e7..dc13ae117 100644
--- a/scripts/formatcode.py
+++ b/scripts/formatcode.py
@@ -7,6 +7,7 @@ import re
 match_expressions = []
 valid_extensions = []
 root_dir = ''
+use_batching = True
 
 def is_header_missing(f):
     with open(f) as reader:
@@ -37,7 +38,11 @@ def scan_tree(root):
                     header_files.append(full_path)
     args = ""
     if files:
-        os.system("clang-format -i " + " ".join(files))
+        if use_batching:
+            os.system("clang-format -i " + " ".join(files))
+        else:
+            for file in files:
+                os.system("clang-format -i " + file)
     if header_files:
         add_headers(header_files, "// Copyright Epic Games, Inc. All Rights Reserved.\n\n")
 
@@ -75,26 +80,31 @@ def parse_match_expressions(wildcards, matches):
         try:
             match_expressions.append(re.compile(regex, re.IGNORECASE))
         except Exception as ex:
-            print('Could not parse input filename expression \'{}\': {}'.format(wildcard, str(ex)))
+            print(f'Could not parse input filename expression \'{wildcard}\': {str(ex)}')
             quit()
     for regex in matches:
         try:
             match_expressions.append(re.compile(regex, re.IGNORECASE))
         except Exception as ex:
-            print('Could not parse input --match expression \'{}\': {}'.format(regex, str(ex)))
+            print(f'Could not parse input --match expression \'{regex}\': {str(ex)}')
             quit()
 
 def _main():
-    global root_dir
+    global root_dir, use_batching
 
     parser = argparse.ArgumentParser()
     parser.add_argument('filenames', nargs='*', help="Match text for filenames. If fullpath contains text it is a match, " +\
         "* is a wildcard. Directory separators are matched by either / or \\. Case insensitive.")
     parser.add_argument('--match', action='append', default=[], help="Match regular expression for filenames. " +\
         "Relative path from the root zen directory must be a complete match. Directory separators are matched only by /. Case insensitive.")
+    parser.add_argument('--batch', dest='use_batching', action='store_true', help="Enable batching calls to clang-format.")
+    parser.add_argument('--no-batch', dest='use_batching', action='store_false', help="Disable batching calls to clang-format.")
+    parser.set_defaults(use_batching=True)
     options = parser.parse_args()
+
     parse_match_expressions(options.filenames, options.match)
     root_dir = pathlib.Path(__file__).parent.parent.resolve()
+    use_batching = options.use_batching
 
     while True:
         if (os.path.isfile(".clang-format")):
diff --git a/scripts/remote_build.py b/scripts/remote_build.py
new file mode 100644
index 000000000..c5787f635
--- /dev/null
+++ b/scripts/remote_build.py
@@ -0,0 +1,270 @@
+import os
+import sys
+import argparse
+import subprocess
+from pathlib import Path
+
+# {{{1 misc --------------------------------------------------------------------
+
+# Disables output of ANSI codes if the terminal doesn't support them
+if os.name == "nt":
+    from ctypes import windll, c_int, byref
+    stdout_handle = windll.kernel32.GetStdHandle(c_int(-11))
+    mode = c_int(0)
+    windll.kernel32.GetConsoleMode(c_int(stdout_handle), byref(mode))
+    ansi_on = (mode.value & 4) != 0
+else:
+    ansi_on = True
+
+#-------------------------------------------------------------------------------
+def _header(*args, ansi=96):
+    if ansi_on:
+        print(f"\x1b[{ansi}m##", *args, end="")
+        print("\x1b[0m")
+    else:
+        print("\n##", *args)
+
+#-------------------------------------------------------------------------------
+def _run_checked(cmd, *args, **kwargs):
+    _header(cmd, *args, ansi=97)
+    ret = subprocess.run((cmd, *args), **kwargs, bufsize=0)
+    if ret.returncode:
+        raise RuntimeError("Failed running " + str(cmd))
+
+#-------------------------------------------------------------------------------
+def _get_ip():
+    import socket
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        s.connect(("172.31.255.255", 1))
+        return s.getsockname()[0]
+    except:
+        return "127.0.0.1"
+    finally:
+        s.close()
+
+#-------------------------------------------------------------------------------
+def _find_binary(name):
+    name += ".exe" if os.name == "nt" else ""
+    for prefix in os.getenv("PATH", "").split(os.pathsep):
+        path = Path(prefix) / name
+        if path.is_file():
+            return path
+    raise EnvironmentError(f"Unable to find '{name}' in the path")
+
+#-------------------------------------------------------------------------------
+class _AutoKill(object):
+    def __init__(self, proc):
+        self._proc = proc
+
+    def __del__(self):
+        self._proc.kill()
+        self._proc.wait()
+        pass
+
+
+
+# {{{1 local -------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+def _local(args):
+    # Parse arguments
+    desc = "Build Zen on a remote host"
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument("remotehost", help="")
+    parser.add_argument("action", default="build", nargs="?", help="")
+    parser.add_argument("--keyfile", default=None, help="SSH key file")
+    args = parser.parse_args(args)
+
+    # Find the binaries we'll need
+    _header("Finding tools")
+    git_bin = _find_binary("git")
+    print(f"Using git from '{git_bin.name}' from '{git_bin.parent}'")
+
+    def find_git_tool(git_bin, tool_name):
+        print(f"Locating {tool_name}...")
+        tool_suffix  = "usr/bin/" + tool_name
+        tool_suffix += ".exe" if os.name == "nt" else ""
+        for parent in git_bin.parents:
+            tool_path = parent / tool_suffix
+            if tool_path.is_file():
+                return tool_path
+        return _find_binary(tool_name)
+
+    ssh_bin = find_git_tool(git_bin, "ssh")
+    scp_bin = find_git_tool(git_bin, "scp")
+    print(f"Using '{ssh_bin.name}' from '{ssh_bin.parent}'")
+    print(f"Using '{scp_bin.name}' from '{scp_bin.parent}'")
+
+    # Find the Zen repository root
+    for parent in Path(__file__).resolve().parents:
+        if (parent / ".git").is_dir():
+            zen_dir = parent
+            break;
+    else:
+        raise EnvironmentError("Unable to find '.git/' directory")
+
+    _header("Validating remote host and credentials")
+
+    # Validate key file. OpenSSL needs a trailing EOL, LibreSSL doesn't
+    if args.keyfile:
+        with open(args.keyfile, "rt") as key_file:
+            lines = [x for x in key_file]
+            if not lines[-1].endswith("\n"):
+                print("!! ERROR: key file must end with a new line")
+                return 1
+        identity = ("-i", args.keyfile)
+    else:
+        identity = ()
+
+    # Validate remote host
+    host = args.remotehost
+    if host == "linux": host = os.getenv("ZEN_REMOTE_HOST_LINUX", "arn-lin-12345")
+    if host == "mac":   host = os.getenv("ZEN_REMOTE_HOST_MAC",   "imacpro-arn.local")
+    """
+    keygen_bin = find_git_tool(git_bin, "ssh-keygen")
+    print(f"Using '{keygen_bin.name}' from '{keygen_bin.parent}'")
+    known_host = subprocess.run((keygen_bin, "-F", host)).returncode
+    if not known_host:
+        print("Adding", host, "as a known host")
+        print("ANSWER 'yes'")
+        known_host = subprocess.run((ssh_bin, *identity, "zenbuild@" + host, "uname -a")).returncode
+        raise IndexError
+    """
+    host = "zenbuild@" + host
+    print(f"Using host '{host}'")
+
+    # Start a git daemon to use as a transfer mechanism
+    _header("Starting a git daemon")
+    print("Port: 4493")
+    print("Base-path: ", zen_dir)
+    print("Host: ", _get_ip())
+    daemon = subprocess.Popen(
+        ( git_bin,
+          "daemon",
+          "--port=4493",
+          "--export-all",
+          "--reuseaddr",
+          "--verbose",
+          "--informative-errors",
+          "--base-path=" + str(zen_dir) ),
+        #stdout = daemon_log,
+        stderr = subprocess.STDOUT
+    )
+    daemon_killer = _AutoKill(daemon)
+
+    # Run this script on the remote machine
+    _header("Running SSH")
+
+    remote_zen_dir = "%s_%s" % (os.getlogin(), _get_ip())
+    print(f"Using zen '~/{remote_zen_dir}'")
+
+    print(f"Running {__file__} remotely")
+    with open(__file__, "rt") as self_file:
+        _run_checked(
+            ssh_bin,
+            *identity,
+            "-tA",
+            host,
+            f"python3 -u - !remote {_get_ip()} '{remote_zen_dir}' main '{args.action}'",
+            stdin=self_file)
+
+    # If we're bundling, collect zip files from the remote machine
+    if args.action == "bundle":
+        build_dir = zen_dir / "build"
+        build_dir.mkdir(exist_ok=True)
+        scp_args = (*identity, host + f":zen/{remote_zen_dir}/build/*.zip", build_dir)
+        _run_checked("scp", *scp_args)
+
+
+
+# {{{1 remote ------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+def _remote(args):
+    # Parse arguments
+    desc = "Build Zen on a remote host"
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument("ip", help="Host's IP address")
+    parser.add_argument("reponame", help="Repository name clone into and work in")
+    parser.add_argument("branch", help="Zen branch to operate on")
+    parser.add_argument("action", help="The action to do")
+    args = parser.parse_args(args)
+
+    # Homeward bound and out
+    zen_dir = Path().home() / "zen"
+    os.chdir(zen_dir)
+
+    # Mutual exclusion
+    """
+    lock_path = zen_dir / "../.remote_lock"
+    try: lock_file = open(lock_path, "xb")
+    except: raise RuntimeError("Failed to lock", lock_path)
+    """
+
+    # Check for a clone, create it, chdir to it
+    _header("REMOTE:", f"Clone/pull from {args.ip}")
+    clone_dir = zen_dir / args.reponame
+    if not clone_dir.is_dir():
+        _run_checked("git", "clone", f"git://{args.ip}:4493/", clone_dir)
+    os.chdir(clone_dir)
+
+    _run_checked("git", "checkout", args.branch)
+    _run_checked("git", "pull", "-r")
+
+    _header("REMOTE:", f"Performing action '{args.action}'")
+
+    # Find xmake
+    xmake_bin = max(x for x in (zen_dir / "xmake").glob("*"))
+    xmake_bin /= "usr/local/bin/xmake"
+
+    # Run xmake
+    xmake_env = {}
+    xmake_env["VCPKG_ROOT"] = zen_dir / "vcpkg"
+    if sys.platform == "linux":
+        xmake_env["CXX"] = "g++-11"
+    print("xmake environment:")
+    for key, value in xmake_env.items():
+        print(" ", key, "=", value)
+    xmake_env.update(os.environ)
+
+    def run_xmake(*args):
+        print("starting xmake...", end="\r")
+        _run_checked(xmake_bin, args[0], "--yes", *args[1:], env=xmake_env)
+
+    if args.action.startswith("build"):
+        mode = "debug" if args.action == "build.debug" else "release"
+        run_xmake("config", "--mode=" + mode)
+        run_xmake("build")
+
+    elif args.action == "bundle":
+        run_xmake("bundle")
+
+    elif args.action == "test":
+        run_xmake("config", "--mode=debug")
+        run_xmake("test")
+
+    elif args.action == "clean":
+        _run_checked("git", "reset")
+        _run_checked("git", "checkout", ".")
+        _run_checked("git", "clean", "-xdf")
+
+
+
+# {{{1 entry -------------------------------------------------------------------
+if __name__ == "__main__":
+    if "!remote" in sys.argv[1:2]:
+        ret = _remote(sys.argv[2:])
+        raise SystemExit(ret)
+
+    try:
+        ret = _local(sys.argv[1:])
+        raise SystemExit(ret)
+    except:
+        raise
+    finally:
+        # Roundabout way to avoid orphaned git-daemon processes
+        if os.name == "nt":
+            os.system("taskkill /f /im git-daemon.exe")
+
+# vim: expandtab foldlevel=1 foldmethod=marker