From df5c556212958e31fcf0c35de60e92d51fb0795c Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 30 Jan 2023 09:47:06 +0100 Subject: Fix Clang warning about bitwise ops on boolean values --- zencore/include/zencore/compactbinary.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h index eba4a1694..c0cd07172 100644 --- a/zencore/include/zencore/compactbinary.h +++ b/zencore/include/zencore/compactbinary.h @@ -367,7 +367,7 @@ public: static constexpr inline bool MayContainAttachments(CbFieldType Type) { // The use of !! will suppress V792 from static analysis. Using //-V792 did not work. - return !!IsObject(Type) | !!IsArray(Type) | !!IsAttachment(Type); + return int(IsObject(Type) == true) | int(IsArray(Type) == true) | int(IsAttachment(Type) == true); } }; -- cgit v1.2.3 From 0bb53f311cb9692ed84bb5fcecd34631ba1997a4 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 30 Jan 2023 09:48:30 +0100 Subject: Call gettid syscall directly as gettid() is not available in earlier glibc --- zencore/thread.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zencore/thread.cpp b/zencore/thread.cpp index 77b75bae3..1597a7dd9 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -29,6 +29,7 @@ # include # include # include +# include # include # include # include @@ -1065,7 +1066,7 @@ GetCurrentThreadId() #if ZEN_PLATFORM_WINDOWS return ::GetCurrentThreadId(); #elif ZEN_PLATFORM_LINUX - return int(gettid()); + return int(syscall(SYS_gettid)); #elif ZEN_PLATFORM_MAC return int(pthread_mach_thread_np(pthread_self())); #endif -- cgit v1.2.3 From 48fd13c2b94522789466a90bd18751b45af6dd0b Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 30 Jan 2023 09:49:21 +0100 Subject: Removed unused add_link() call The blake3 library is now provided by vcpkg and marked as a dependency via the add_package() call. --- zencore/xmake.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/zencore/xmake.lua b/zencore/xmake.lua index aae20274b..61000f739 100644 --- a/zencore/xmake.lua +++ b/zencore/xmake.lua @@ -14,7 +14,6 @@ target('zencore') add_includedirs("include", {public=true}) add_includedirs("$(projectdir)/thirdparty/utfcpp/source") add_includedirs("$(projectdir)/thirdparty/trace", {public=true}) - add_links("blake3") if is_os("windows") then add_linkdirs("$(projectdir)/thirdparty/Oodle/lib/Win64") elseif is_os("linux") then -- cgit v1.2.3 From 40d1744cfb4acf2fcd3c03cc56ce5236ba48047e Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 30 Jan 2023 10:12:46 +0100 Subject: Missing pthread dependency --- zentest-appstub/xmake.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zentest-appstub/xmake.lua b/zentest-appstub/xmake.lua index d43b05028..d8e0283c1 100644 --- a/zentest-appstub/xmake.lua +++ b/zentest-appstub/xmake.lua @@ -5,6 +5,10 @@ target("zentest-appstub") add_headerfiles("**.h") add_files("*.cpp") + if is_os("linux") then + add_syslinks("pthread") + end + if is_plat("macosx") then add_ldflags("-framework CoreFoundation") add_ldflags("-framework Security") -- cgit v1.2.3 From 7daa8765688e784a7b0da532514ee28bf1127c6f Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 30 Jan 2023 10:18:11 +0100 Subject: Fixed symbol resolution error because xmake reverses dependent libraries --- zenserver/xmake.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index 8b18bf9f9..5bccbcbeb 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -45,3 +45,11 @@ target("zenserver") "vcpkg::sentry-native", "vcpkg::sol2" ) + + if is_plat("linux") then + -- As sentry_native uses symbols from breakpad_client, the latter must + -- be specified after the former with GCC-like toolchains. xmake however + -- is unaware of this and simply globs files from vcpkg's output. The + -- line below forces breakpad_client to be to the right of sentry_native + add_syslinks("breakpad_client") + end -- cgit v1.2.3 From 995b9371dbd567d0ea77fc969bacccb7cd64dc4f Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 2 Feb 2023 14:42:28 +0100 Subject: Scripts to acquire and use UE's Linux toolchain --- scripts/ue_build_linux/README.md | 31 +++++++++++++++ scripts/ue_build_linux/clang | 2 + scripts/ue_build_linux/clang++ | 2 + scripts/ue_build_linux/get_ue_toolchain.sh | 64 ++++++++++++++++++++++++++++++ scripts/ue_build_linux/ue_build.sh | 27 +++++++++++++ 5 files changed, 126 insertions(+) create mode 100755 scripts/ue_build_linux/README.md create mode 100755 scripts/ue_build_linux/clang create mode 100755 scripts/ue_build_linux/clang++ create mode 100755 scripts/ue_build_linux/get_ue_toolchain.sh create mode 100755 scripts/ue_build_linux/ue_build.sh diff --git a/scripts/ue_build_linux/README.md b/scripts/ue_build_linux/README.md new file mode 100755 index 000000000..8cd589610 --- /dev/null +++ b/scripts/ue_build_linux/README.md @@ -0,0 +1,31 @@ +# Build Zen with the UE Linux toolchain + +This folder contains scripts to build Zen using the UE Linux toolchain. It +works by using the --sysroot= option to redirect compilers and linkers to find +headers and libraries. There are a few components involved; + +1) get_ue_toolchain.sh + +This will download the required components from Perforce and struicture them in +such a way that they can be used by both vcpkg and xmake when building Zen. + +2) ue_build.sh [args...] + +Given the toolchain location downloaded in step (1), this script sets up a +suitable environment and execs the "prog [args...]". It is expected that this +is used to invoke xmake to build Zen; + +``` +$ scripts/ue_linux_build/ue_build.sh ~/uetools xmake config --mode=debug --vcpkg=~/vcpkg +$ scripts/ue_linux_build/ue_build.sh ~/uetools xmake build +``` + +It is possible that `--toolchain=clang` may be required as a configuration +option. The ue_build.sh script can also be source into the current shell,, +although it is worth noting that this has never been tried. + +3) clang / clang++ + +These acts as shims to the binaries in `toolchain_dir`, adding in the required +command line arguments to use the correct headers and libraries. The ue_build.sh +script adjusts $PATH appropriately. diff --git a/scripts/ue_build_linux/clang b/scripts/ue_build_linux/clang new file mode 100755 index 000000000..f3af44c9c --- /dev/null +++ b/scripts/ue_build_linux/clang @@ -0,0 +1,2 @@ +#!/bin/sh +exec $UE_TOOLCHAIN_DIR/bin/clang --sysroot=$UE_TOOLCHAIN_DIR $CFLAGS $* diff --git a/scripts/ue_build_linux/clang++ b/scripts/ue_build_linux/clang++ new file mode 100755 index 000000000..d2ebda549 --- /dev/null +++ b/scripts/ue_build_linux/clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +exec $UE_TOOLCHAIN_DIR/bin/clang++ --sysroot=$UE_TOOLCHAIN_DIR -stdlib=libc++ $CXXFLAGS $* diff --git a/scripts/ue_build_linux/get_ue_toolchain.sh b/scripts/ue_build_linux/get_ue_toolchain.sh new file mode 100755 index 000000000..67fd922c2 --- /dev/null +++ b/scripts/ue_build_linux/get_ue_toolchain.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +die() { echo "ERROR: $1"; exit; } + +if [ -z $1 ]; then + echo "usage: $(basename ${BASH_SOURCE[0]}) " + exit +fi + +if [ -e $1 ]; then + rmdir $1 + if [ $? -gt 0 ]; then + die "$1 is not empty" + exit + fi +fi + +mkdir -p $1 +cd $1 + +p4 login -s +if [ $? -gt 0 ]; then + die "Not logged into Perforce" +fi + +# Perforce paths +if [ -z $AUTOSDK_PATH ]; then + AUTOSDK_PATH=//depot/CarefullyRedist/HostLinux/Linux_x64/v21_clang-15.0.1-centos7/x86_64-unknown-linux-gnu +fi + +if [ -z $UE_PATH ]; then + UE_PATH=//UE5/Main +fi + +if [ -z $UE_LIBCXX_PATH ]; then + UE_LIBCXX_PATH=$UE_PATH/Engine/Source/ThirdParty/Unix/LibCxx +fi + +p4_print() { + echo -n $1 + p4 print -q -o$2/$(basename $1) $1 + echo +} + +# toolchain +p4_print $AUTOSDK_PATH/bin/clang* bin +#p4_print $AUTOSDK_PATH/bin/lld bin + +p4_print $AUTOSDK_PATH/lib/gcc/... lib/gcc +p4_print $AUTOSDK_PATH/lib/clang/*/include/... lib/clang/*/include + +# headers +p4_print $AUTOSDK_PATH/usr/include/...h usr/include + +# libraries +p4_print $AUTOSDK_PATH/lib64/* lib64 +p4_print $AUTOSDK_PATH/usr/lib64/* usr/lib64 + +mkdir -p usr/lib +mv usr/lib64/*.o usr/lib + +# ue's libc++ +p4_print $UE_LIBCXX_PATH/include/... include +p4_print $UE_LIBCXX_PATH/lib/Unix/x86_64-unknown-linux-gnu/* lib64 diff --git a/scripts/ue_build_linux/ue_build.sh b/scripts/ue_build_linux/ue_build.sh new file mode 100755 index 000000000..a15028560 --- /dev/null +++ b/scripts/ue_build_linux/ue_build.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +die() { echo ERROR: $1; exit 1; } + +# Validate input +if [ -z $1 ]; then + echo "usage: $0 " + exit 1 +fi + +if ! [ -d $1 ]; then + die "$1 is not a directory" +fi + +if [ -z $1 ]; then + die "$1/bin/clang++ does not exist" +fi + +export UE_TOOLCHAIN_DIR=$(realpath $1) +export CC="clang" +export CXX="clang++" +export LD="clang++" + +export PATH="$(realpath $(dirname ${BASH_SOURCE[0]})):$PATH" + +shift +exec $* -- cgit v1.2.3 From 2fdb7d8b22b3c7f9eec583c1861d09293484971d Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 2 Feb 2023 14:43:46 +0100 Subject: Removed size_t override that is no longer required with newer libc++ --- zencore/include/zencore/compactbinarybuilder.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h index dbd010705..5f6d9fd0c 100644 --- a/zencore/include/zencore/compactbinarybuilder.h +++ b/zencore/include/zencore/compactbinarybuilder.h @@ -539,15 +539,6 @@ operator<<(CbWriter& Writer, std::nullptr_t) return Writer; } -#if defined(_LIBCPP_VERSION) -inline CbWriter& -operator<<(CbWriter& Writer, std::size_t Value) -{ - Writer.AddInteger(uint64_t(Value)); - return Writer; -} -#endif - inline CbWriter& operator<<(CbWriter& Writer, std::wstring_view Value) { -- cgit v1.2.3 From af806aae3cacc750ee62bcdfcce2752b31502106 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 2 Feb 2023 14:51:07 +0100 Subject: Removed the AppImage-based bundling step on Linux --- scripts/bundle.lua | 4 +--- scripts/bundle_linux.sh | 37 ------------------------------------- 2 files changed, 1 insertion(+), 40 deletions(-) delete mode 100755 scripts/bundle_linux.sh diff --git a/scripts/bundle.lua b/scripts/bundle.lua index ef63df0fb..b2857e42f 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -263,11 +263,9 @@ local function main_linux() _append_content_zip("build/linux/x86_64/release/zenserver") _append_content_zip("build/linux/x86_64/release/zen") - _exec("scripts/bundle_linux.sh") - _zip(false, "build/zenserver-linux.zip", - "build/appimage/zenserver", + "build/linux/x86_64/release/zenserver", "build/linux/x86_64/release/zen", crashpad_handler_path) end diff --git a/scripts/bundle_linux.sh b/scripts/bundle_linux.sh deleted file mode 100755 index baefafbaa..000000000 --- a/scripts/bundle_linux.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -# Create and change to the directory we'll do all the work in -work_dir=$(dirname $0)/../build/appimage -mkdir -p $work_dir -cd $work_dir - -# First we'll build a filesystem that contains zenserver -mkdir fs - -# Copy the main binary -zs_bin=../linux/x86_64/release/zenserver -mkdir fs/bin -cp $zs_bin fs/bin/ - -# Include the linked C++ library .so - this is required because some LTS distros -# do not include GCC-11 at the time of writing and Zen's uses this for C++20. -mkdir fs/usr -cp $(ldd $zs_bin |sed -nr 's/^.+\s+=>\s+(.+)\s+\(0x0000.+$/\1/p'|grep 'c++') fs/usr/ - -# Download AppImage's bootstrap binary and create .desktop configuratino for it -unlink AppRun-x86_64 2>/dev/null -wget "https://github.com/AppImage/AppImageKit/releases/download/13/AppRun-x86_64" -mv AppRun-x86_64 fs/AppRun -chmod a+x fs/AppRun -echo Exec=zenserver>fs/zenserver.desktop - -# Create the filesystem and concatenate with AppImage's runtime. -mksquashfs fs zenserver.fs -root-owned -noappend - -unlink runtime-x86_64 2>/dev/null -wget "https://github.com/AppImage/AppImageKit/releases/download/13/runtime-x86_64" - -unlink zenserver 2>/dev/null -cat runtime-x86_64 >> zenserver -cat zenserver.fs >> zenserver -chmod a+x zenserver \ No newline at end of file -- cgit v1.2.3 From acfe1ef59d8d5aee9f9b137fa9a8b4effec9af3b Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Fri, 3 Feb 2023 08:41:18 +0100 Subject: Commented out breakpad_client xmake.lua adjustment - not applicable yet Later baselines of vcpkg will change to a newer version of sentry-native that will produce libbreakpad_client. This block of xmake script will then be required to coerce xmake to order libraries correctly --- zenserver/xmake.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index 5bccbcbeb..992923d48 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -46,6 +46,8 @@ target("zenserver") "vcpkg::sol2" ) + -- Only applicable to later versions of sentry-native + --[[ if is_plat("linux") then -- As sentry_native uses symbols from breakpad_client, the latter must -- be specified after the former with GCC-like toolchains. xmake however @@ -53,3 +55,4 @@ target("zenserver") -- line below forces breakpad_client to be to the right of sentry_native add_syslinks("breakpad_client") end + ]]-- -- cgit v1.2.3 From 9e18f34e7d6b32f96cfa2456d7c7ea2ab5d6f123 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Fri, 3 Feb 2023 10:20:36 +0100 Subject: Added a size_t override for Apple Clang where types as size_t != uint64_t --- zencore/include/zencore/compactbinarybuilder.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h index 5f6d9fd0c..5311bbb07 100644 --- a/zencore/include/zencore/compactbinarybuilder.h +++ b/zencore/include/zencore/compactbinarybuilder.h @@ -539,6 +539,17 @@ operator<<(CbWriter& Writer, std::nullptr_t) return Writer; } +#if defined(__clang__) && defined(__APPLE__) +/* Apple Clang has different types for uint64_t and size_t so an override is + needed here. Without it, Clang can't disambiguate integer overloads */ +inline CbWriter& +operator<<(CbWriter& Writer, std::size_t Value) +{ + Writer.AddInteger(uint64_t(Value)); + return Writer; +} +#endif + inline CbWriter& operator<<(CbWriter& Writer, std::wstring_view Value) { -- cgit v1.2.3 From 5b998227c82ce92dda547984be3a20e84fde815a Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Fri, 3 Feb 2023 10:37:02 +0100 Subject: Removed stale comment now that use of !! has been replaced --- zencore/include/zencore/compactbinary.h | 1 - 1 file changed, 1 deletion(-) diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h index c0cd07172..b546f97aa 100644 --- a/zencore/include/zencore/compactbinary.h +++ b/zencore/include/zencore/compactbinary.h @@ -366,7 +366,6 @@ public: /** Whether the type is or may contain fields of any attachment type. */ static constexpr inline bool MayContainAttachments(CbFieldType Type) { - // The use of !! will suppress V792 from static analysis. Using //-V792 did not work. return int(IsObject(Type) == true) | int(IsArray(Type) == true) | int(IsAttachment(Type) == true); } }; -- cgit v1.2.3 From 1a3db6fc978a3de752fa001213927617508d3aa7 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 6 Feb 2023 14:09:36 +0100 Subject: Fixed two typos in Readme --- scripts/ue_build_linux/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ue_build_linux/README.md b/scripts/ue_build_linux/README.md index 8cd589610..1665b24a2 100755 --- a/scripts/ue_build_linux/README.md +++ b/scripts/ue_build_linux/README.md @@ -6,7 +6,7 @@ headers and libraries. There are a few components involved; 1) get_ue_toolchain.sh -This will download the required components from Perforce and struicture them in +This will download the required components from Perforce and structure them in such a way that they can be used by both vcpkg and xmake when building Zen. 2) ue_build.sh [args...] @@ -21,7 +21,7 @@ $ scripts/ue_linux_build/ue_build.sh ~/uetools xmake build ``` It is possible that `--toolchain=clang` may be required as a configuration -option. The ue_build.sh script can also be source into the current shell,, +option. The ue_build.sh script can also be sourced into the current shell, although it is worth noting that this has never been tried. 3) clang / clang++ -- cgit v1.2.3 From aa07aca9dc95ffb14b3dfaa788f22461fe0e9863 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 6 Feb 2023 14:10:08 +0100 Subject: Added note about UE toolchain meeting VFX reference platform versions --- scripts/ue_build_linux/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/ue_build_linux/README.md b/scripts/ue_build_linux/README.md index 1665b24a2..00249ddbe 100755 --- a/scripts/ue_build_linux/README.md +++ b/scripts/ue_build_linux/README.md @@ -1,8 +1,9 @@ # Build Zen with the UE Linux toolchain -This folder contains scripts to build Zen using the UE Linux toolchain. It -works by using the --sysroot= option to redirect compilers and linkers to find -headers and libraries. There are a few components involved; +This folder contains scripts to build Zen using the UE Linux toolchain. This +can be used to output binaries that meet the VFX Reference Platform versions. +It works by using the --sysroot= option to redirect compilers and linkers to +find headers and libraries. There are a few components involved; 1) get_ue_toolchain.sh -- cgit v1.2.3 From 9c8d24e111f5bc69bff60dfb88fe32add7579fdb Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 6 Feb 2023 14:10:36 +0100 Subject: Fixed wrong if-statement when checking for the presence of clang++ --- scripts/ue_build_linux/ue_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ue_build_linux/ue_build.sh b/scripts/ue_build_linux/ue_build.sh index a15028560..e9c3b2cf6 100755 --- a/scripts/ue_build_linux/ue_build.sh +++ b/scripts/ue_build_linux/ue_build.sh @@ -12,7 +12,7 @@ if ! [ -d $1 ]; then die "$1 is not a directory" fi -if [ -z $1 ]; then +if ! [ -e $1/bin/clang++ ]; then die "$1/bin/clang++ does not exist" fi -- cgit v1.2.3