revive UE toolchain build (#343)

* add comment about 7z usage in xmake bundle
* add ue-libcxx lib/headers and licence + tps
* update get_ue_toolchain.sh to use embedded libc++ and decouple from p4
* clearer output when falling back to zip from 7z
* update ci scripts to use ue toolchain on linux
* updated linux build README.md
* changelog

3 files changed, 43 insertions, 49 deletions

diff --git a/scripts/bundle.lua b/scripts/bundle.lua
index 30ce9554b..a8a6b973a 100644
--- a/scripts/bundle.lua
+++ b/scripts/bundle.lua
@@ -103,7 +103,8 @@ local function _zip(store_only, zip_path, ...)
         end
         return
     end
-    print("7z not found")
+
+    print("7z not found, falling back to zip")
 
     import("detect.tools.find_zip")
     zip_cmd = find_zip()
diff --git a/scripts/ue_build_linux/README.md b/scripts/ue_build_linux/README.md
index 00249ddbe..060f1a77a 100755
--- a/scripts/ue_build_linux/README.md
+++ b/scripts/ue_build_linux/README.md
@@ -2,31 +2,38 @@
 

 This folder contains scripts to build Zen using the UE Linux toolchain. This

 can be used to output binaries that meet the VFX Reference Platform versions.

-It works by using the --sysroot= option to redirect compilers and linkers to

+It works by using the `--sysroot=` option to redirect compilers and linkers to

 find headers and libraries. There are a few components involved;

 

 1) get_ue_toolchain.sh <toolchain_dir>

 

-This will download the required components from Perforce and structure them in

-such a way that they can be used by both vcpkg and xmake when building Zen.

+```

+$ scripts/ue_linux_build/get_ue_toolchain.sh ./.tmp-ue-toolchain

+```

+

+This will download the required components from cdn.unrealengine.com and

+structure them in such a way that they can be used by both vcpkg and xmake

+when building Zen.

 

 2) ue_build.sh <toolchain_dir> <prog> [args...] 

 

-Given the toolchain location downloaded in step (1), this script sets up a

-suitable environment and execs the "prog [args...]". It is expected that this

-is used to invoke xmake to build Zen;

+Given the toolchain location downloaded in step (1) and the `VCPKG_ROOT`

+environment variable is properly configured, this script sets up a suitable

+environment and execs the "prog [args...]".

+

+It is expected that this is used to invoke xmake to build Zen;

 

 ```

-$ scripts/ue_linux_build/ue_build.sh ~/uetools xmake config --mode=debug --vcpkg=~/vcpkg

-$ scripts/ue_linux_build/ue_build.sh ~/uetools xmake build

+$ scripts/ue_linux_build/ue_build.sh .tmp-ue-toolchain xmake config --mode=debug

+$ scripts/ue_linux_build/ue_build.sh .tmp-ue-toolchain xmake build

 ```

 

 It is possible that `--toolchain=clang` may be required as a configuration

-option. The ue_build.sh script can also be sourced into the current shell,

+option. The `ue_build.sh` script can also be sourced into the current shell,

 although it is worth noting that this has never been tried.

 

-3) clang / clang++

+3) `scripts/ue_build_linux/clang` / `scripts/ue_build_linux/clang++`

 

 These acts as shims to the binaries in `toolchain_dir`, adding in the required

-command line arguments to use the correct headers and libraries. The ue_build.sh

-script adjusts $PATH appropriately.

+command line arguments to use the correct headers and libraries.

+The `ue_build.sh` script adjusts `$PATH` appropriately.

diff --git a/scripts/ue_build_linux/get_ue_toolchain.sh b/scripts/ue_build_linux/get_ue_toolchain.sh
index 67fd922c2..f4a1b58d4 100755
--- a/scripts/ue_build_linux/get_ue_toolchain.sh
+++ b/scripts/ue_build_linux/get_ue_toolchain.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ZEN_ROOT=$(realpath $SCRIPT_DIR/../..)
+
 die() { echo "ERROR: $1"; exit; }
 
 if [ -z $1 ]; then
@@ -18,47 +21,30 @@ fi
 mkdir -p $1
 cd $1
 
-p4 login -s
-if [ $? -gt 0 ]; then
-	die "Not logged into Perforce"
-fi
-
-# Perforce paths
-if [ -z $AUTOSDK_PATH ]; then
-	AUTOSDK_PATH=//depot/CarefullyRedist/HostLinux/Linux_x64/v21_clang-15.0.1-centos7/x86_64-unknown-linux-gnu
-fi
-
-if [ -z $UE_PATH ]; then
-	UE_PATH=//UE5/Main
-fi
-
-if [ -z $UE_LIBCXX_PATH ]; then
-	UE_LIBCXX_PATH=$UE_PATH/Engine/Source/ThirdParty/Unix/LibCxx
-fi
+mkdir -p tmp
+cd tmp
 
-p4_print() {
-	echo -n $1
-	p4 print -q -o$2/$(basename $1) $1
-	echo
-}
+#CLANG_VERSION=v21_clang-15.0.1-centos7
+CLANG_VERSION=v22_clang-16.0.6-centos7
+TOOLCHAIN_PLATFORM=x86_64-unknown-linux-gnu
 
-# toolchain
-p4_print $AUTOSDK_PATH/bin/clang* bin
-#p4_print $AUTOSDK_PATH/bin/lld bin
+echo "Fetching UE toolchain $CLANG_VERSION..."
+wget -q --show-progress http://cdn.unrealengine.com/Toolchain_Linux/native-linux-$CLANG_VERSION.tar.gz
 
-p4_print $AUTOSDK_PATH/lib/gcc/... lib/gcc
-p4_print $AUTOSDK_PATH/lib/clang/*/include/... lib/clang/*/include
+echo "Extracting toolchain $TOOLCHAIN_PLATFORM..."
+tar -xzf native-linux-$CLANG_VERSION.tar.gz $CLANG_VERSION/$TOOLCHAIN_PLATFORM/
+mv $CLANG_VERSION/$TOOLCHAIN_PLATFORM/* ..
 
-# headers
-p4_print $AUTOSDK_PATH/usr/include/...h usr/include
-
-# libraries
-p4_print $AUTOSDK_PATH/lib64/* lib64
-p4_print $AUTOSDK_PATH/usr/lib64/* usr/lib64
+cd ..
+rm -rf tmp
 
+rm -rf usr/lib
 mkdir -p usr/lib
 mv usr/lib64/*.o usr/lib
 
-# ue's libc++
-p4_print $UE_LIBCXX_PATH/include/... include
-p4_print $UE_LIBCXX_PATH/lib/Unix/x86_64-unknown-linux-gnu/* lib64
+echo "Fetching UE headers and libc++ from $ZEN_ROOT/thirdparty/ue-libcxx..."
+cp -r $ZEN_ROOT/thirdparty/ue-libcxx/include/* ./include
+mkdir -p ./lib64
+cp -r $ZEN_ROOT/thirdparty/ue-libcxx/lib64/* ./lib64
+
+echo "Done"