diff options
| -rw-r--r-- | .github/workflows/create_release.yml | 4 | ||||
| -rw-r--r-- | .github/workflows/validate.yml | 6 | ||||
| -rw-r--r-- | README.md | 68 | ||||
| -rw-r--r-- | scripts/bundle.lua | 10 | ||||
| -rwxr-xr-x | scripts/ue_build_linux/README.md | 42 | ||||
| -rwxr-xr-x | scripts/ue_build_linux/clang | 2 | ||||
| -rwxr-xr-x | scripts/ue_build_linux/clang++ | 2 | ||||
| -rwxr-xr-x | scripts/ue_build_linux/get_ue_toolchain.sh | 2 | ||||
| -rwxr-xr-x | scripts/ue_build_linux/ue_build.sh | 33 | ||||
| -rw-r--r-- | toolchains/ue_clang.lua | 60 | ||||
| -rw-r--r-- | toolchains/xmake.lua | 3 | ||||
| -rw-r--r-- | xmake.lua | 42 |
12 files changed, 164 insertions, 110 deletions
diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 163c0cf85..2eeedc0c2 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -60,11 +60,11 @@ jobs: - name: Config run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake config -v -y -m release --zensentry=yes + xmake config -v -y -m release --zensentry=yes --toolchain=ue-clang --sdk=${{ github.workspace }}/.tmp-ue-toolchain - name: Bundle run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake bundle -v -y + xmake bundle -v -y - name: Get Sentry CLI run: | diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 32b75fe5e..d96645ac9 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -145,7 +145,7 @@ jobs: - name: Config run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} --zensentry=yes + xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} --zensentry=yes --toolchain=ue-clang --sdk=${{ github.workspace }}/.tmp-ue-toolchain - name: Clean reports if: ${{ matrix.config == 'debug' }} @@ -155,7 +155,7 @@ jobs: - name: Build & Test if: ${{ matrix.config == 'debug' }} run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake test -v -y --junit + xmake test -v -y --junit - name: Upload report if: ${{ (failure() || success()) && (matrix.config == 'debug') }} @@ -176,7 +176,7 @@ jobs: - name: Bundle if: ${{ matrix.config == 'release' }} run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake bundle -v -y + xmake bundle -v -y - name: Upload zenserver-linux if: ${{ github.ref_name == 'main' && matrix.config == 'release' }} @@ -75,60 +75,56 @@ line. ## Building on Linux -The following instructions have been collated using Ubuntu 20.04. - -Zen makes use of C++20 features which at the time of writing has limited -toolchain and C++ library support. A minimum compiler version of GCC-11 or -Clang-12 is required, along with GNU's libstdc+++-11 or newer. - -The first step is to acquire a suitable compiler and C++ library. On Ubuntu 20.04 -GCC-11 is not available in the standard package repositories so Ubuntu's tool- -chain test respository needs to be added to Apt (note that this is _not required_ -for Ubuntu 21.04 onwards); - -``` -sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test -``` - -Now GCC-11 can be installed via Apt. This will also install a suitable version of -the C++ library. - -``` -sudo apt install -y --no-install-recommends g++-11 -g++-11 --version -``` - -The easiest way to tell `xmake` to use the correct compiler version is to set -the `CXX` environment variable. +The following instructions have been collated using Ubuntu 24.04. Other distributions will +likely need a slightly different set of packages installed. -``` -export CXX=g++-11 +```bash +sudo apt install -y build-essential pkg-config unzip ``` Install [xmake](https://xmake.io/#/getting_started). -``` +It's important to use the correct version when installing, pay attention to the last argument in the command line below: + +```bash curl -fsSL https://xmake.io/shget.text | bash -s v2.9.9 ``` -Clone the Zen project and tell `xmake` to use the correct GCC version. +Clone the Zen project. -``` +```bash git clone https://github.com/EpicGames/zen.git ~/zen/main cd ~/zen/main ``` -Now we are ready to build Zen. The `-y` skips `xmake` from prompting about -building packages. +We recommend building using the UE Linux toolchain, to match the Unreal Editor +setup as closely as possible. It is also the most thoroughly tested toolchain +since this is what is used in the CI environment which produces the binaries +which end up in the UE tree. + +Using other toolchains may or may not work depending on which version and +compiler you happen to be using. +### Building with the UE Linux toolchain + +To build with the Unreal Engine cross-compile toolchain (producing binaries +compatible with the VFX Reference Platform), first download the toolchain: + +```bash +scripts/ue_build_linux/get_ue_toolchain.sh ``` -xmake config -y --mode=debug + +This downloads to `~/.ue-toolchain` by default. xmake automatically detects the +toolchain at this location, so no extra flags are needed: + +```bash +xmake config -y -m debug xmake build ``` -Note that the command above to set the build variant to debug is optional. Tests -are only built in debug.The `xmake` flags `-vD` can be useful to diagnose -`xmake` issues. +The toolchain can also be selected explicitly with `--toolchain=ue-clang`, and +the SDK location can be overridden with `--sdk=<path>` or the +`UE_TOOLCHAIN_DIR` environment variable. ## Building on Mac diff --git a/scripts/bundle.lua b/scripts/bundle.lua index 07e120d04..b2a5e1e08 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -17,7 +17,15 @@ end -------------------------------------------------------------------------------- local function _build(arch, debug, config_args) + import("core.project.config") + config.load() + variant = debug and "debug" or "release" + + -- Preserve toolchain/sdk from current config so --clean doesn't lose them + local toolchain_arg = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or nil + local sdk_arg = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or nil + local ret = _exec( "xmake", "config", @@ -26,6 +34,8 @@ local function _build(arch, debug, config_args) "--mode="..variant, "--arch="..arch, "--zensentry=yes", + toolchain_arg, + sdk_arg, config_args) if ret > 0 then raise("Failed to configure xmake") diff --git a/scripts/ue_build_linux/README.md b/scripts/ue_build_linux/README.md index e93a234ae..afafcbe24 100755 --- a/scripts/ue_build_linux/README.md +++ b/scripts/ue_build_linux/README.md @@ -2,38 +2,38 @@ This folder contains scripts to build Zen using the UE Linux toolchain. This
can be used to output binaries that meet the VFX Reference Platform versions.
-It works by using the `--sysroot=` option to redirect compilers and linkers to
-find headers and libraries. There are a few components involved;
-1) get_ue_toolchain.sh <toolchain_dir>
+## Setup
+
+Download the toolchain using `get_ue_toolchain.sh`:
```
-$ scripts/ue_build_linux/get_ue_toolchain.sh ./.tmp-ue-toolchain
+$ scripts/ue_build_linux/get_ue_toolchain.sh
```
-This will download the required components from cdn.unrealengine.com and
-structure them in such a way that they can be used by both vcpkg and xmake
-when building Zen.
+By default this downloads to `~/.ue-toolchain`. A custom path can be given as
+the first argument, or via the `UE_TOOLCHAIN_DIR` environment variable.
-2) ue_build.sh <toolchain_dir> <prog> [args...]
+This will download the required components from cdn.unrealengine.com and
+structure them in such a way that they can be used by xmake when building Zen.
-Given the toolchain location downloaded in step (1) and the `VCPKG_ROOT`
-environment variable is properly configured, this script sets up a suitable
-environment and execs the "prog [args...]".
+## Building
-It is expected that this is used to invoke xmake to build Zen;
+xmake automatically detects the toolchain at `~/.ue-toolchain`, so no extra
+flags are needed:
```
-$ scripts/ue_build_linux/ue_build.sh .tmp-ue-toolchain xmake config --mode=debug
-$ scripts/ue_build_linux/ue_build.sh .tmp-ue-toolchain xmake build
+$ xmake config -y -m debug
+$ xmake build -y
```
-It is possible that `--toolchain=clang` may be required as a configuration
-option. The `ue_build.sh` script can also be sourced into the current shell,
-although it is worth noting that this has never been tried.
+To build a release bundle:
-3) `scripts/ue_build_linux/clang` / `scripts/ue_build_linux/clang++`
+```
+$ xmake config -y -m release
+$ xmake bundle -y
+```
-These acts as shims to the binaries in `toolchain_dir`, adding in the required
-command line arguments to use the correct headers and libraries.
-The `ue_build.sh` script adjusts `$PATH` appropriately.
+The toolchain can also be selected explicitly with `--toolchain=ue-clang`, and
+the SDK location can be overridden with `--sdk=<path>` (must be absolute) or
+the `UE_TOOLCHAIN_DIR` environment variable.
diff --git a/scripts/ue_build_linux/clang b/scripts/ue_build_linux/clang deleted file mode 100755 index 9666ba4ba..000000000 --- a/scripts/ue_build_linux/clang +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec $UE_TOOLCHAIN_DIR/bin/clang --sysroot=$UE_TOOLCHAIN_DIR $CFLAGS "$@" diff --git a/scripts/ue_build_linux/clang++ b/scripts/ue_build_linux/clang++ deleted file mode 100755 index be106ae87..000000000 --- a/scripts/ue_build_linux/clang++ +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec $UE_TOOLCHAIN_DIR/bin/clang++ --sysroot=$UE_TOOLCHAIN_DIR -stdlib=libc++ $CXXFLAGS "$@" diff --git a/scripts/ue_build_linux/get_ue_toolchain.sh b/scripts/ue_build_linux/get_ue_toolchain.sh index c2538b09a..0afd40fe3 100755 --- a/scripts/ue_build_linux/get_ue_toolchain.sh +++ b/scripts/ue_build_linux/get_ue_toolchain.sh @@ -5,7 +5,7 @@ ZEN_ROOT=$(realpath $SCRIPT_DIR/../..) die() { echo "ERROR: $1"; exit; } -toolchain_dir="${1:-${ZEN_ROOT}/.tmp-ue-toolchain}" +toolchain_dir="${1:-${UE_TOOLCHAIN_DIR:-${HOME}/.ue-toolchain}}" if [[ $toolchain_dir == "--help" ]]; then echo "usage: $(basename ${BASH_SOURCE[0]}) <output_dir>" diff --git a/scripts/ue_build_linux/ue_build.sh b/scripts/ue_build_linux/ue_build.sh deleted file mode 100755 index 690f9f661..000000000 --- a/scripts/ue_build_linux/ue_build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -ZEN_ROOT=$(realpath $SCRIPT_DIR/../..) - -die() { echo ERROR: $1; exit 1; } - -if [[ $1 == "--help" ]]; then - echo "usage: $0 <ue_toolchain_dir> (defaults to ${ZEN_ROOT}/.tmp-ue-toolchain)" - exit -fi - -toolchain_dir="${1:-${ZEN_ROOT}/.tmp-ue-toolchain}" - -# Validate input - -if ! [ -d $toolchain_dir ]; then - die "$1 is not a directory" -fi - -if ! [ -e $toolchain_dir/bin/clang++ ]; then - die "$1/bin/clang++ does not exist" -fi - -export UE_TOOLCHAIN_DIR=$(realpath $toolchain_dir) -export CC="clang" -export CXX="clang++" -export LD="clang++" - -export PATH="$(realpath $(dirname ${BASH_SOURCE[0]})):$PATH" - -shift -exec "$@" diff --git a/toolchains/ue_clang.lua b/toolchains/ue_clang.lua new file mode 100644 index 000000000..8a1efcc04 --- /dev/null +++ b/toolchains/ue_clang.lua @@ -0,0 +1,60 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +-- Custom toolchain for building with the Unreal Engine Linux clang toolchain. +-- +-- Usage: +-- xmake config --toolchain=ue-clang [--sdk=/path/to/ue-toolchain] +-- +-- If --sdk is not given, the toolchain is resolved in order: +-- 1. $UE_TOOLCHAIN_DIR environment variable +-- 2. ~/.ue-toolchain (default get_ue_toolchain.sh download location) + +toolchain("ue-clang") + set_kind("standalone") + + on_load(function (toolchain) + -- Resolve SDK directory: --sdk flag > $UE_TOOLCHAIN_DIR > ~/.ue-toolchain + local sdkdir = toolchain:sdkdir() + if not sdkdir or sdkdir == "" then + sdkdir = os.getenv("UE_TOOLCHAIN_DIR") + end + if not sdkdir or sdkdir == "" then + local default_path = path.join(os.getenv("HOME"), ".ue-toolchain") + if os.isdir(default_path) then + sdkdir = default_path + end + end + if not sdkdir or sdkdir == "" then + return + end + + toolchain:config_set("sdkdir", sdkdir) + + local bindir = path.join(sdkdir, "bin") + + -- Compiler and linker tools + toolchain:set("toolset", "cc", path.join(bindir, "clang")) + toolchain:set("toolset", "cxx", path.join(bindir, "clang++")) + toolchain:set("toolset", "ld", path.join(bindir, "clang++")) + toolchain:set("toolset", "sh", path.join(bindir, "clang++")) + toolchain:set("toolset", "ar", path.join(bindir, "llvm-ar")) + toolchain:set("toolset", "strip", path.join(bindir, "llvm-strip")) + toolchain:set("toolset", "objcopy", path.join(bindir, "llvm-objcopy")) + toolchain:set("toolset", "as", path.join(bindir, "clang")) + toolchain:set("toolset", "ranlib", path.join(bindir, "llvm-ranlib")) + + -- Sysroot and stdlib flags for both C and C++ + local sysroot_flag = "--sysroot=" .. sdkdir + toolchain:add("cxflags", sysroot_flag, {force = true}) + toolchain:add("cxflags", "-stdlib=libc++", {force = true}) + + -- Linker flags: sysroot and stdlib selection only. + -- Static libc++ overrides are applied at project level (xmake.lua) so they + -- don't leak into cmake package builds (e.g. sentry-native/crashpad). + toolchain:add("ldflags", sysroot_flag, {force = true}) + toolchain:add("ldflags", "-stdlib=libc++", {force = true}) + + toolchain:add("shflags", sysroot_flag, {force = true}) + toolchain:add("shflags", "-stdlib=libc++", {force = true}) + end) +toolchain_end() diff --git a/toolchains/xmake.lua b/toolchains/xmake.lua new file mode 100644 index 000000000..2817a1586 --- /dev/null +++ b/toolchains/xmake.lua @@ -0,0 +1,3 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +includes("ue_clang.lua") @@ -101,16 +101,17 @@ if is_plat("windows") then add_requires("7z") end --- If we're using the UE cross-compile toolchain, we need to ensure we link statically --- against the toolchain libc++ and libc++abi, as the system ones can differ in ABI --- leading to nasty crashes - -if is_plat("linux") and os.getenv("UE_TOOLCHAIN_DIR") then - add_ldflags("-static-libstdc++") - add_ldflags("$(projectdir)/thirdparty/ue-libcxx/lib64/libc++.a") - add_ldflags("$(projectdir)/thirdparty/ue-libcxx/lib64/libc++abi.a") - set_toolset("objcopy", "$(env UE_TOOLCHAIN_DIR)/bin/llvm-objcopy") +-- When using the UE clang toolchain, statically link the toolchain's libc++ and +-- libc++abi to avoid ABI mismatches with system libraries at runtime. +-- These are project-level flags (not in the toolchain definition) so they don't +-- propagate into cmake package builds. +if is_plat("linux") and get_config("toolchain") == "ue-clang" then + add_ldflags("-static-libstdc++", {force = true}) + add_ldflags("$(projectdir)/thirdparty/ue-libcxx/lib64/libc++.a", {force = true}) + add_ldflags("$(projectdir)/thirdparty/ue-libcxx/lib64/libc++abi.a", {force = true}) + add_ldflags("-lpthread", {force = true}) end + if has_config("zensentry") and not use_asan then if is_plat("linux") then add_requires("sentry-native 0.12.1", {configs = {backend = "crashpad"}}) @@ -276,6 +277,25 @@ set_languages("cxx20") -- always generate debug information set_symbols("debug") +includes("toolchains") + +-- Auto-select the UE clang toolchain on Linux when the SDK is available +if is_plat("linux") and not get_config("toolchain") then + local ue_sdk = os.getenv("UE_TOOLCHAIN_DIR") + if not ue_sdk or ue_sdk == "" then + local home = os.getenv("HOME") + if home then + local default_path = path.join(home, ".ue-toolchain") + if os.isdir(default_path) then + ue_sdk = default_path + end + end + end + if ue_sdk and ue_sdk ~= "" and os.isdir(ue_sdk) then + set_toolchains("ue-clang") + end +end + includes("thirdparty") includes("src/transports") includes("src/zenbase") @@ -459,7 +479,9 @@ task("test") -- Only reconfigure if current config doesn't already match if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch then - os.exec("xmake config -c -m debug -p %s -a %s", plat, arch) + local toolchain_flag = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or "" + local sdk_flag = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or "" + os.exec("xmake config -c -m debug -p %s -a %s %s %s", plat, arch, toolchain_flag, sdk_flag) end -- Build targets we're going to run |