Merge pull request #219 from EpicGames/linux-build

Scripts for building Linux binaries using UE's Linux toolchain.

13 files changed, 149 insertions, 45 deletions

diff --git a/scripts/bundle.lua b/scripts/bundle.lua
index ef63df0fb..b2857e42f 100644
--- a/scripts/bundle.lua
+++ b/scripts/bundle.lua
@@ -263,11 +263,9 @@ local function main_linux()
     _append_content_zip("build/linux/x86_64/release/zenserver")
     _append_content_zip("build/linux/x86_64/release/zen")
 
-    _exec("scripts/bundle_linux.sh")
-
     _zip(false,
         "build/zenserver-linux.zip",
-        "build/appimage/zenserver",
+        "build/linux/x86_64/release/zenserver",
         "build/linux/x86_64/release/zen",
         crashpad_handler_path)
 end
diff --git a/scripts/bundle_linux.sh b/scripts/bundle_linux.sh
deleted file mode 100755
index baefafbaa..000000000
--- a/scripts/bundle_linux.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/sh
-
-# Create and change to the directory we'll do all the work in
-work_dir=$(dirname $0)/../build/appimage
-mkdir -p $work_dir
-cd $work_dir
-
-# First we'll build a filesystem that contains zenserver
-mkdir fs
-
-# Copy the main binary
-zs_bin=../linux/x86_64/release/zenserver
-mkdir fs/bin
-cp $zs_bin fs/bin/
-
-# Include the linked C++ library .so - this is required because some LTS distros
-# do not include GCC-11 at the time of writing and Zen's uses this for C++20.
-mkdir fs/usr
-cp $(ldd $zs_bin |sed -nr 's/^.+\s+=>\s+(.+)\s+\(0x0000.+$/\1/p'|grep 'c++') fs/usr/
-
-# Download AppImage's bootstrap binary and create .desktop configuratino for it
-unlink AppRun-x86_64 2>/dev/null
-wget "https://github.com/AppImage/AppImageKit/releases/download/13/AppRun-x86_64"
-mv AppRun-x86_64 fs/AppRun
-chmod a+x fs/AppRun
-echo Exec=zenserver>fs/zenserver.desktop
-
-# Create the filesystem and concatenate with AppImage's runtime.
-mksquashfs fs zenserver.fs -root-owned -noappend
-
-unlink runtime-x86_64 2>/dev/null
-wget "https://github.com/AppImage/AppImageKit/releases/download/13/runtime-x86_64"
-
-unlink zenserver 2>/dev/null
-cat runtime-x86_64 >> zenserver
-cat zenserver.fs >> zenserver
-chmod a+x zenserver
\ No newline at end of file
diff --git a/scripts/ue_build_linux/README.md b/scripts/ue_build_linux/README.md
new file mode 100755
index 000000000..00249ddbe
--- /dev/null
+++ b/scripts/ue_build_linux/README.md
@@ -0,0 +1,32 @@
+# Build Zen with the UE Linux toolchain

+

+This folder contains scripts to build Zen using the UE Linux toolchain. This

+can be used to output binaries that meet the VFX Reference Platform versions.

+It works by using the --sysroot= option to redirect compilers and linkers to

+find headers and libraries. There are a few components involved;

+

+1) get_ue_toolchain.sh <toolchain_dir>

+

+This will download the required components from Perforce and structure them in

+such a way that they can be used by both vcpkg and xmake when building Zen.

+

+2) ue_build.sh <toolchain_dir> <prog> [args...] 

+

+Given the toolchain location downloaded in step (1), this script sets up a

+suitable environment and execs the "prog [args...]". It is expected that this

+is used to invoke xmake to build Zen;

+

+```

+$ scripts/ue_linux_build/ue_build.sh ~/uetools xmake config --mode=debug --vcpkg=~/vcpkg

+$ scripts/ue_linux_build/ue_build.sh ~/uetools xmake build

+```

+

+It is possible that `--toolchain=clang` may be required as a configuration

+option. The ue_build.sh script can also be sourced into the current shell,

+although it is worth noting that this has never been tried.

+

+3) clang / clang++

+

+These acts as shims to the binaries in `toolchain_dir`, adding in the required

+command line arguments to use the correct headers and libraries. The ue_build.sh

+script adjusts $PATH appropriately.

diff --git a/scripts/ue_build_linux/clang b/scripts/ue_build_linux/clang
new file mode 100755
index 000000000..f3af44c9c
--- /dev/null
+++ b/scripts/ue_build_linux/clang
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec $UE_TOOLCHAIN_DIR/bin/clang --sysroot=$UE_TOOLCHAIN_DIR $CFLAGS $*
diff --git a/scripts/ue_build_linux/clang++ b/scripts/ue_build_linux/clang++
new file mode 100755
index 000000000..d2ebda549
--- /dev/null
+++ b/scripts/ue_build_linux/clang++
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec $UE_TOOLCHAIN_DIR/bin/clang++ --sysroot=$UE_TOOLCHAIN_DIR -stdlib=libc++ $CXXFLAGS $*
diff --git a/scripts/ue_build_linux/get_ue_toolchain.sh b/scripts/ue_build_linux/get_ue_toolchain.sh
new file mode 100755
index 000000000..67fd922c2
--- /dev/null
+++ b/scripts/ue_build_linux/get_ue_toolchain.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+die() { echo "ERROR: $1"; exit; }
+
+if [ -z $1 ]; then
+	echo "usage: $(basename ${BASH_SOURCE[0]}) <output_dir>"
+	exit
+fi
+
+if [ -e $1 ]; then
+	rmdir $1
+	if [ $? -gt 0 ]; then
+		die "$1 is not empty"
+		exit
+	fi
+fi
+
+mkdir -p $1
+cd $1
+
+p4 login -s
+if [ $? -gt 0 ]; then
+	die "Not logged into Perforce"
+fi
+
+# Perforce paths
+if [ -z $AUTOSDK_PATH ]; then
+	AUTOSDK_PATH=//depot/CarefullyRedist/HostLinux/Linux_x64/v21_clang-15.0.1-centos7/x86_64-unknown-linux-gnu
+fi
+
+if [ -z $UE_PATH ]; then
+	UE_PATH=//UE5/Main
+fi
+
+if [ -z $UE_LIBCXX_PATH ]; then
+	UE_LIBCXX_PATH=$UE_PATH/Engine/Source/ThirdParty/Unix/LibCxx
+fi
+
+p4_print() {
+	echo -n $1
+	p4 print -q -o$2/$(basename $1) $1
+	echo
+}
+
+# toolchain
+p4_print $AUTOSDK_PATH/bin/clang* bin
+#p4_print $AUTOSDK_PATH/bin/lld bin
+
+p4_print $AUTOSDK_PATH/lib/gcc/... lib/gcc
+p4_print $AUTOSDK_PATH/lib/clang/*/include/... lib/clang/*/include
+
+# headers
+p4_print $AUTOSDK_PATH/usr/include/...h usr/include
+
+# libraries
+p4_print $AUTOSDK_PATH/lib64/* lib64
+p4_print $AUTOSDK_PATH/usr/lib64/* usr/lib64
+
+mkdir -p usr/lib
+mv usr/lib64/*.o usr/lib
+
+# ue's libc++
+p4_print $UE_LIBCXX_PATH/include/... include
+p4_print $UE_LIBCXX_PATH/lib/Unix/x86_64-unknown-linux-gnu/* lib64
diff --git a/scripts/ue_build_linux/ue_build.sh b/scripts/ue_build_linux/ue_build.sh
new file mode 100755
index 000000000..e9c3b2cf6
--- /dev/null
+++ b/scripts/ue_build_linux/ue_build.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+die() { echo ERROR: $1; exit 1; }
+
+# Validate input
+if [ -z $1 ]; then
+	echo "usage: $0 <ue_toolchain_dir>"
+	exit 1
+fi
+
+if ! [ -d $1 ]; then
+	die "$1 is not a directory"
+fi
+
+if ! [ -e $1/bin/clang++ ]; then
+	die "$1/bin/clang++ does not exist"
+fi
+
+export UE_TOOLCHAIN_DIR=$(realpath $1)
+export CC="clang"
+export CXX="clang++"
+export LD="clang++"
+
+export PATH="$(realpath $(dirname ${BASH_SOURCE[0]})):$PATH"
+
+shift
+exec $*
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
index eba4a1694..b546f97aa 100644
--- a/zencore/include/zencore/compactbinary.h
+++ b/zencore/include/zencore/compactbinary.h
@@ -366,8 +366,7 @@ public:
 	/** Whether the type is or may contain fields of any attachment type. */
 	static constexpr inline bool MayContainAttachments(CbFieldType Type)
 	{
-		// The use of !! will suppress V792 from static analysis. Using //-V792 did not work.
-		return !!IsObject(Type) | !!IsArray(Type) | !!IsAttachment(Type);
+		return int(IsObject(Type) == true) | int(IsArray(Type) == true) | int(IsAttachment(Type) == true);
 	}
 };
 
diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h
index dbd010705..5311bbb07 100644
--- a/zencore/include/zencore/compactbinarybuilder.h
+++ b/zencore/include/zencore/compactbinarybuilder.h
@@ -539,7 +539,9 @@ operator<<(CbWriter& Writer, std::nullptr_t)
 	return Writer;
 }
 
-#if defined(_LIBCPP_VERSION)
+#if defined(__clang__) && defined(__APPLE__)
+/* Apple Clang has different types for uint64_t and size_t so an override is
+   needed here. Without it, Clang can't disambiguate integer overloads */
 inline CbWriter&
 operator<<(CbWriter& Writer, std::size_t Value)
 {
diff --git a/zencore/thread.cpp b/zencore/thread.cpp
index 77b75bae3..1597a7dd9 100644
--- a/zencore/thread.cpp
+++ b/zencore/thread.cpp
@@ -29,6 +29,7 @@
 #	include <sys/file.h>
 #	include <sys/sem.h>
 #	include <sys/stat.h>
+#	include <sys/syscall.h>
 #	include <sys/wait.h>
 #	include <time.h>
 #	include <unistd.h>
@@ -1065,7 +1066,7 @@ GetCurrentThreadId()
 #if ZEN_PLATFORM_WINDOWS
 	return ::GetCurrentThreadId();
 #elif ZEN_PLATFORM_LINUX
-	return int(gettid());
+	return int(syscall(SYS_gettid));
 #elif ZEN_PLATFORM_MAC
 	return int(pthread_mach_thread_np(pthread_self()));
 #endif
diff --git a/zencore/xmake.lua b/zencore/xmake.lua
index aae20274b..61000f739 100644
--- a/zencore/xmake.lua
+++ b/zencore/xmake.lua
@@ -14,7 +14,6 @@ target('zencore')
     add_includedirs("include", {public=true})
     add_includedirs("$(projectdir)/thirdparty/utfcpp/source")
     add_includedirs("$(projectdir)/thirdparty/trace", {public=true})
-    add_links("blake3")
     if is_os("windows") then
         add_linkdirs("$(projectdir)/thirdparty/Oodle/lib/Win64")
     elseif is_os("linux") then
diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua
index 8b18bf9f9..992923d48 100644
--- a/zenserver/xmake.lua
+++ b/zenserver/xmake.lua
@@ -45,3 +45,14 @@ target("zenserver")
         "vcpkg::sentry-native",
         "vcpkg::sol2"
     )
+
+	-- Only applicable to later versions of sentry-native
+	--[[
+    if is_plat("linux") then
+        -- As sentry_native uses symbols from breakpad_client, the latter must
+        -- be specified after the former with GCC-like toolchains. xmake however
+        -- is unaware of this and simply globs files from vcpkg's output. The
+        -- line below forces breakpad_client to be to the right of sentry_native
+        add_syslinks("breakpad_client")
+    end
+	]]--
diff --git a/zentest-appstub/xmake.lua b/zentest-appstub/xmake.lua
index d43b05028..d8e0283c1 100644
--- a/zentest-appstub/xmake.lua
+++ b/zentest-appstub/xmake.lua
@@ -5,6 +5,10 @@ target("zentest-appstub")
     add_headerfiles("**.h")
     add_files("*.cpp")
 
+    if is_os("linux") then
+        add_syslinks("pthread")
+    end
+
     if is_plat("macosx") then
         add_ldflags("-framework CoreFoundation")
         add_ldflags("-framework Security")