diff options
| author | Per Larsson <[email protected]> | 2022-03-19 10:41:08 +0100 |
|---|---|---|
| committer | Per Larsson <[email protected]> | 2022-03-19 10:41:08 +0100 |
| commit | 703c252710cdae35a35be700fade144e994777c0 (patch) | |
| tree | 03268a46ea32553c4777ee30cc617f140665938a | |
| parent | Minor cleanup. (diff) | |
| parent | Suppress C4305 in third party includes (diff) | |
| download | zen-703c252710cdae35a35be700fade144e994777c0.tar.xz zen-703c252710cdae35a35be700fade144e994777c0.zip | |
Merge branch 'main' into streamapi
28 files changed, 1212 insertions, 374 deletions
diff --git a/.clang-format b/.clang-format index f1b47c7d7..4688f60ae 100644 --- a/.clang-format +++ b/.clang-format @@ -31,7 +31,7 @@ BraceWrapping: AfterCaseLabel: true AfterClass: true AfterControlStatement: Always - AfterEnum: false + AfterEnum: true AfterFunction: true AfterNamespace: false AfterObjCDeclaration: false @@ -66,10 +66,6 @@ DerivePointerAlignment: false DisableFormat: false ExperimentalAutoDetectBinPacking: false FixNamespaceComments: true -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH IncludeBlocks: Preserve IncludeCategories: - Regex: '^<ext/.*\.h>' @@ -118,31 +114,7 @@ PointerAlignment: Left RawStringFormats: - Language: Cpp Delimiters: - - cc - - CC - cpp - - Cpp - - CPP - - 'c++' - - 'C++' - CanonicalDelimiter: '' - BasedOnStyle: google - - Language: TextProto - Delimiters: - - pb - - PB - - proto - - PROTO - EnclosingFunctions: - - EqualsProto - - EquivToProto - - PARSE_PARTIAL_TEXT_PROTO - - PARSE_TEST_PROTO - - PARSE_TEXT_PROTO - - ParseTextOrDie - - ParseTextProtoOrDie - - ParseTestProto - - ParsePartialTestProto CanonicalDelimiter: '' BasedOnStyle: google ReflowComments: true @@ -169,14 +141,9 @@ SpacesInSquareBrackets: false SpaceBeforeSquareBrackets: false BitFieldColonSpacing: Both Standard: Auto -StatementMacros: - - Q_UNUSED - - QT_REQUIRE_VERSION TabWidth: 4 UseCRLF: false UseTab: Always WhitespaceSensitiveMacros: - STRINGIZE - - PP_STRINGIZE - - BOOST_PP_STRINGIZE ... @@ -96,8 +96,8 @@ Next we need the `xmake` build system. For this we will download and install date `.deb` files; ``` -wget https://github.com/xmake-io/xmake/releases/download/v2.6.2/xmake-v2.6.2.amd64.deb -sudo dpkg -i xmake-v2.5.7.amd64.deb +wget https://github.com/xmake-io/xmake/releases/download/v2.6.4/xmake-v2.6.4.amd64.deb +sudo dpkg -i xmake-v2.6.4.amd64.deb xmake --version ``` @@ -136,6 +136,23 @@ Note that the command above to set the build variant to debug is optional. Tests are only built in debug.The `xmake` flags `-vD` can be useful to diagnose `xmake` issues. +### Distribution compatibility + +Builds for Linux have a dependency on a modern version of the C++ library that +supports Zen's use of C++20 and the GCC-11 toolchain used (assuming the above +build instructions are adhered to). However there is no guarantee that the this +dependency is met when building on one Linux install and running on another. For +example, at the time of writing the LTS version of Ubuntu and Debian do not have +GCC-11's version of libstdc++.so.6 available in their package repositories. + +To solve this, `xmake bundle` will bundle the required C++ .so and zenserver +binary together using AppImage. This can briefly be summarised as create a +squashfs file system containing zenserver and any C++ shared objects list with +`lld`. This file system is then concatenated with AppImage's runtime. This is +implemented in `scripts/bundle_linux.sh`. More details can be found here; + +https://github.com/AppImage/AppImageKit + ## Building on Mac Building on Mac is very similar to Linux; install xmake, clone vcpkg and Zen and diff --git a/scripts/bundle.lua b/scripts/bundle.lua index 90fb13657..01268ab14 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -2,8 +2,15 @@ -------------------------------------------------------------------------------- local function _exec(cmd, ...) - print("--", cmd, ...) - local ret = os.execv(cmd, {...}) + local args = {} + for _, arg in pairs({...}) do + if arg then + table.insert(args, arg) + end + end + + print("--", cmd, table.unpack(args)) + local ret = os.execv(cmd, args) print() return ret end @@ -35,23 +42,52 @@ local function _build(arch, debug, config_args) end -------------------------------------------------------------------------------- -local function _zip(zip_path, ...) +local function _zip(store_only, zip_path, ...) + -- Here's the rules; if len(...) is 1 and it is a dir then create a zip with + -- archive paths like this; + -- + -- glob(foo/bar/**) -> foo/bar/abc, foo/bar/dir/123 -> zip(abc, dir/123) + -- + -- Otherwise assume ... is file paths and add without leading directories; + -- + -- foo/abc, bar/123 -> zip(abc, 123) + + zip_path = path.absolute(zip_path) os.tryrm(zip_path) + local inputs = {...} + + local source_dir = nil + if #inputs == 1 and os.isdir(inputs[1]) then + source_dir = inputs[1] + end + import("detect.tools.find_7z") local cmd = find_7z() if cmd then - -- A ./ prefix makes 7z ignore the directory structure input_paths = {} - for _, input_path in ipairs({...}) do - input_path = path.relative(input_path, ".") - if input_path:sub(2,2) ~= ":" then - input_path = "./"..input_path + if source_dir then + -- Suffixing a directory path with a "/." will have 7z set the path + -- for archived files relative to that directory. + input_paths = { path.join(source_dir, ".") } + else + for _, input_path in pairs(inputs) do + -- If there is a "/./" anywhere in file paths then 7z drops all + -- directory information and just archives the file by name + input_path = path.relative(input_path, ".") + if input_path:sub(2,2) ~= ":" then + input_path = "./"..input_path + end + table.insert(input_paths, input_path) end - table.insert(input_paths, input_path) end - local ret = _exec("7z", "a", zip_path, table.unpack(input_paths)) + compression_level = "-mx9" + if store_only then + compression_level = "-mx0" + end + + local ret = _exec("7z", "a", "-r", compression_level, zip_path, table.unpack(input_paths)) if ret > 0 then raise("Received error from 7z") end @@ -62,10 +98,29 @@ local function _zip(zip_path, ...) import("detect.tools.find_zip") cmd = find_zip() if cmd then - local ret = _exec("zip", "--junk-paths", zip_path, ...) + local input_paths = inputs + local cwd = os.curdir() + if source_dir then + os.cd(source_dir) + input_paths = { "." } + end + + compression_level = "-9" + if store_only then + compression_level = "-0" + end + + local strip_leading_path = nil + if not source_dir then + strip_leading_path = "--junk-paths" + end + + local ret = _exec("zip", "-r", compression_level, strip_leading_path, zip_path, table.unpack(input_paths)) if ret > 0 then raise("Received error from zip") end + + os.cd(cwd) return end print("zip not found") @@ -90,6 +145,21 @@ local function _find_vcpkg_binary(triple, port, binary) end -------------------------------------------------------------------------------- +local function _append_content_zip(bin_path) + local zip_path = "build/frontend.zip" + local content_dir = "zenserver/frontend/html/" + _zip(true, zip_path, content_dir) + + zip_file = io.open(zip_path, "rb") + local zip_data = zip_file:read("*all") + zip_file:close() + + bin_file = io.open(bin_path, "ab") + bin_file:write(zip_data) + bin_file:close() +end + +-------------------------------------------------------------------------------- local function main_windows() import("core.base.option") @@ -102,12 +172,14 @@ local function main_windows() _build("x64", false, config_args) + _append_content_zip("build/windows/x64/release/zenserver.exe") + local crashpad_handler_path = _find_vcpkg_binary( "x64-windows-static", "sentry-native", "crashpad_handler.exe") - _zip( + _zip(false, zip_path, "build/windows/x64/release/zenserver.exe", "build/windows/x64/release/zenserver.pdb", @@ -129,9 +201,11 @@ local function main_mac() "build/macosx/arm64/release/zenserver" ) if ret > 0 then - raise("Failed creating univeral binary") + raise("Failed creating universal binary") end + _append_content_zip("build/macosx/universal/release/zenserver") + -- At the time of writing vcpkg does not support sentry-native on arm64. Once -- it does we can create a univeral binary for this. For now just bundle x64 local crashpad_handler_path = _find_vcpkg_binary( @@ -140,7 +214,7 @@ local function main_mac() "crashpad_handler") -- Zip - _zip( + _zip(false, "build/zenserver-macos.zip", "build/macosx/universal/release/zenserver", crashpad_handler_path) @@ -159,9 +233,11 @@ local function main_linux() "crashpad_handler") --]] + _append_content_zip("build/linux/x86_64/release/zenserver") + _exec("scripts/bundle_linux.sh") - _zip( + _zip(false, "build/zenserver-linux.zip", "build/appimage/zenserver", crashpad_handler_path) diff --git a/scripts/formatcode.py b/scripts/formatcode.py index dc13ae117..49a8753da 100644 --- a/scripts/formatcode.py +++ b/scripts/formatcode.py @@ -1,8 +1,9 @@ import argparse -import os import fileinput +import os import pathlib import re +import subprocess match_expressions = [] valid_extensions = [] @@ -89,6 +90,20 @@ def parse_match_expressions(wildcards, matches): print(f'Could not parse input --match expression \'{regex}\': {str(ex)}') quit() +def validate_clang_format(): + vstring = subprocess.check_output("clang-format --version", shell=True).decode().rstrip() + + match = re.search(r'(\d+)\.(\d+)(\.(\d+))?$', vstring) + if not match: + raise ValueError("invalid version number '%s'" % vstring) + + (major, minor, patch) = match.group(1, 2, 4) + + if int(major) < 13: + if int(minor) == 0: + if int(patch) < 1: + raise ValueError(f'invalid clang-format version -- we require at least v12.0.1') + def _main(): global root_dir, use_batching @@ -106,6 +121,8 @@ def _main(): root_dir = pathlib.Path(__file__).parent.parent.resolve() use_batching = options.use_batching + validate_clang_format() + while True: if (os.path.isfile(".clang-format")): scan_zen(".") diff --git a/scripts/remote_build.py b/scripts/remote_build.py index 08e44c9ad..70f9cf9bf 100644 --- a/scripts/remote_build.py +++ b/scripts/remote_build.py @@ -106,13 +106,15 @@ def _local(args): _header("Validating remote host and credentials") - # Validate key file. OpenSSL needs a trailing EOL, LibreSSL doesn't + # Validate key file. Git's SSH uses OpenSSL which needs UNIX line-endings if args.keyfile: with open(args.keyfile, "rt") as key_file: - lines = [x for x in key_file] - if not lines[-1].endswith("\n"): - print("!! ERROR: key file must end with a new line") - return 1 + lines = [x.strip() for x in key_file] + + with open(args.keyfile, "wb") as key_file: + for line in lines: + key_file.write(line.encode() + b"\n") + identity = ("-i", args.keyfile) else: identity = () diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index 902ec26c8..7cc6db68a 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -1754,7 +1754,7 @@ TEST_CASE("uson") SUBCASE("CbField(None|Type|Name)") { constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; + const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; CbFieldView NoneField(NoneBytes); CHECK(NoneField.GetSize() == sizeof(NoneBytes)); @@ -1775,7 +1775,7 @@ TEST_CASE("uson") SUBCASE("CbField(None|Type)") { constexpr CbFieldType FieldType = CbFieldType::None; - constexpr const char NoneBytes[] = {char(FieldType)}; + const char NoneBytes[] = {char(FieldType)}; CbFieldView NoneField(NoneBytes); CHECK(NoneField.GetSize() == sizeof NoneBytes); @@ -1792,14 +1792,14 @@ TEST_CASE("uson") SUBCASE("CbField(None|Name)") { constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; + const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; CbFieldView NoneField(NoneBytes + 1, FieldType); CHECK(NoneField.GetSize() == uint64_t(sizeof NoneBytes)); CHECK(NoneField.GetName().compare("Name") == 0); CHECK(NoneField.HasName() == true); CHECK(NoneField.HasValue() == false); CHECK(NoneField.GetHash() == IoHash::HashBuffer(NoneBytes, sizeof NoneBytes)); - CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1)); + CHECK(NoneField.GetView() == MemoryView(NoneBytes + 1, sizeof NoneBytes - 1)); MemoryView SerializedView; CHECK(NoneField.TryGetSerializedView(SerializedView) == false); @@ -1810,15 +1810,15 @@ TEST_CASE("uson") SUBCASE("CbField(None|EmptyName)") { - constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - constexpr const uint8_t NoneBytes[] = {uint8_t(FieldType), 0}; - CbFieldView NoneField(NoneBytes + 1, FieldType); + constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; + const uint8_t NoneBytes[] = {uint8_t(FieldType), 0}; + CbFieldView NoneField(NoneBytes + 1, FieldType); CHECK(NoneField.GetSize() == sizeof NoneBytes); CHECK(NoneField.GetName().empty() == true); CHECK(NoneField.HasName() == true); CHECK(NoneField.HasValue() == false); CHECK(NoneField.GetHash() == IoHash::HashBuffer(NoneBytes, sizeof NoneBytes)); - CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1)); + CHECK(NoneField.GetView() == MemoryView(NoneBytes + 1, sizeof NoneBytes - 1)); MemoryView SerializedView; CHECK(NoneField.TryGetSerializedView(SerializedView) == false); } diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 5dbbb95bf..449236127 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -27,6 +27,10 @@ enum class ZenContentType : uint8_t kCompressedBinary = 7, kUnknownContentType = 8, kHTML = 9, + kJavaScript = 10, + kCSS = 11, + kPNG = 12, + kIcon = 13, kCOUNT }; @@ -58,6 +62,15 @@ ToString(ZenContentType ContentType) return "yaml"sv; case ZenContentType::kHTML: return "html"sv; + case ZenContentType::kJavaScript: + return "javascript"sv; + case ZenContentType::kCSS: + return "css"sv; + case ZenContentType::kPNG: + return "png"sv; + case ZenContentType::kIcon: + return "icon"sv; + } } @@ -336,7 +349,7 @@ public: /** Create a buffer which references a sequence of bytes inside another buffer */ - ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes); + ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes=~0ull); /** Create a buffer which references a range of bytes which we assume will live * for the entire life time. @@ -351,7 +364,7 @@ public: ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); - inline operator bool() const { return !m_Core->IsNull(); } + inline explicit operator bool() const { return !m_Core->IsNull(); } inline operator MemoryView() const& { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); } inline void MakeOwned() { return m_Core->MakeOwned(); } [[nodiscard]] inline bool IsOwned() const { return m_Core->IsOwned(); } diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h index 74ab0f81f..7c2deeed7 100644 --- a/zencore/include/zencore/logging.h +++ b/zencore/include/zencore/logging.h @@ -54,7 +54,7 @@ struct LogCategory #define ZEN_DEFINE_LOG_CATEGORY_STATIC(Category, Name) \ static struct LogCategory##Category : public LogCategory \ { \ - LogCategory##Category() : LogCategory(Name) {} \ + LogCategory##Category() : LogCategory(Name) {} \ } Category; #define ZEN_LOG_TRACE(Category, fmtstr, ...) \ diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index bd5e5a531..60ebd0f5f 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -72,10 +72,11 @@ #ifndef ZEN_THIRD_PARTY_INCLUDES_START # if ZEN_COMPILER_MSC -# define ZEN_THIRD_PARTY_INCLUDES_START \ - __pragma(warning(push)) __pragma(warning(disable : 4668)) /* use of undefined preprocessor macro */ \ - __pragma(warning(disable : 4267)) /* '=': conversion from 'size_t' to 'US' */ \ - __pragma(warning(disable : 4127)) +# define ZEN_THIRD_PARTY_INCLUDES_START \ + __pragma(warning(push)) __pragma(warning(disable : 4668)) /* use of undefined preprocessor macro */ \ + __pragma(warning(disable : 4305)) /* 'if': truncation from 'uint32' to 'bool' */ \ + __pragma(warning(disable : 4267)) /* '=': conversion from 'size_t' to 'US' */ \ + __pragma(warning(disable : 4127)) /* conditional expression is constant */ # elif ZEN_COMPILER_CLANG # define ZEN_THIRD_PARTY_INCLUDES_START \ _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wundef\"") \ diff --git a/zencore/thread.cpp b/zencore/thread.cpp index 7b1396f5f..c2ecc8d72 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -75,7 +75,7 @@ SetNameInternal(DWORD thread_id, const char* name) #endif #if ZEN_PLATFORM_LINUX -const bool bNoZombieChildren = [] () { +const bool bNoZombieChildren = []() { // When a child process exits it is put into a zombie state until the parent // collects its result. This doesn't fit the Windows-like model that Zen uses // where there is a less strict familial model and no zombification. Ignoring @@ -87,7 +87,7 @@ const bool bNoZombieChildren = [] () { Action.sa_handler = SIG_IGN; sigaction(SIGCHLD, &Action, nullptr); return true; -} (); +}(); #endif void @@ -647,10 +647,10 @@ ProcessHandle::Wait(int TimeoutMs) timespec SleepTime = {0, SleepMs * 1000 * 1000}; for (int i = 0;; i += SleepMs) { -#if ZEN_PLATFORM_MAC +# if ZEN_PLATFORM_MAC int WaitState = 0; waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED); -#endif +# endif if (kill(m_Pid, 0) < 0) { diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp index c8e11468e..710b6f356 100644 --- a/zenhttp/httpserver.cpp +++ b/zenhttp/httpserver.cpp @@ -60,6 +60,18 @@ MapContentTypeToString(HttpContentType ContentType) case HttpContentType::kHTML: return "text/html"sv; + + case HttpContentType::kJavaScript: + return "application/javascript"sv; + + case HttpContentType::kCSS: + return "text/css"sv; + + case HttpContentType::kPNG: + return "image/png"sv; + + case HttpContentType::kIcon: + return "image/x-icon"sv; } } @@ -76,6 +88,10 @@ static constinit uint32_t HashCompressedBinary = HashStringDjb2("application/x static constinit uint32_t HashJson = HashStringDjb2("json"sv); static constinit uint32_t HashYaml = HashStringDjb2("yaml"sv); static constinit uint32_t HashHtml = HashStringDjb2("text/html"sv); +static constinit uint32_t HashJavaScript = HashStringDjb2("application/javascript"sv); +static constinit uint32_t HashCss = HashStringDjb2("text/css"sv); +static constinit uint32_t HashPng = HashStringDjb2("image/png"sv); +static constinit uint32_t HashIcon = HashStringDjb2("image/x-icon"sv); std::once_flag InitContentTypeLookup; @@ -96,6 +112,10 @@ struct HashedTypeEntry {HashText, HttpContentType::kText}, {HashCompressedBinary, HttpContentType::kCompressedBinary}, {HashHtml, HttpContentType::kHTML}, + {HashJavaScript, HttpContentType::kJavaScript}, + {HashCss, HttpContentType::kCSS}, + {HashPng, HttpContentType::kPNG}, + {HashIcon, HttpContentType::kIcon}, // clang-format on }; diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index e4d5697fa..044078aa4 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -6,6 +6,7 @@ # include <upstream/jupiter.h> # include <upstream/upstreamapply.h> +# include <upstream/upstreamcache.h> # include <zencore/compactbinary.h> # include <zencore/compactbinarybuilder.h> # include <zencore/compactbinarypackage.h> @@ -173,10 +174,10 @@ SandboxedFunctionJob::GrantNamedObjectAccess(PWSTR ObjectName, SE_OBJECT_TYPE Ob .grfAccessMode = GRANT_ACCESS, .grfInheritance = grfInhericance, .Trustee = {.pMultipleTrustee = nullptr, - .MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE, - .TrusteeForm = TRUSTEE_IS_SID, - .TrusteeType = TRUSTEE_IS_GROUP, - .ptstrName = (PWSTR)m_AppContainerSid}}; + .MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE, + .TrusteeForm = TRUSTEE_IS_SID, + .TrusteeType = TRUSTEE_IS_GROUP, + .ptstrName = (PWSTR)m_AppContainerSid}}; PACL OldAcl = nullptr; @@ -328,24 +329,29 @@ SandboxedFunctionJob::SpawnJob(std::filesystem::path ExePath) //////////////////////////////////////////////////////////////////////////////// -HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& BaseDir) +HttpFunctionService::HttpFunctionService(CasStore& Store, + CidStore& InCidStore, + const std::filesystem::path& BaseDir, + const CloudCacheClientOptions& ComputeOptions, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const UpstreamAuthConfig& StorageAuthConfig, + AuthMgr& Mgr) : m_Log(logging::Get("apply")) , m_CasStore(Store) , m_CidStore(InCidStore) , m_SandboxPath(BaseDir / "scratch") , m_FunctionPath(BaseDir / "func") { - m_UpstreamApply = MakeUpstreamApply({}, m_CasStore, m_CidStore); - - CloudCacheAccessToken AccessToken{.Value = "ServiceAccount 0f8056b30bd0df0959be55fc3338159b6f938456d3474aed0087fb965268d079", - .ExpireTime = CloudCacheAccessToken::TimePoint::max()}; - - CloudCacheClientOptions Options = {.ServiceUrl = "https://horde.devtools-dev.epicgames.com"sv, - .DdcNamespace = "default"sv, - .BlobStoreNamespace = "default"sv}; - - auto HordeUpstreamEndpoint = - MakeHordeUpstreamEndpoint(Options, CloudCacheTokenProvider::CreateFromStaticToken(AccessToken), m_CasStore, m_CidStore); + m_UpstreamApply = UpstreamApply::Create({}, m_CasStore, m_CidStore); + + auto HordeUpstreamEndpoint = UpstreamApplyEndpoint::CreateHordeEndpoint(ComputeOptions, + ComputeAuthConfig, + StorageOptions, + StorageAuthConfig, + m_CasStore, + m_CidStore, + Mgr); m_UpstreamApply->RegisterEndpoint(std::move(HordeUpstreamEndpoint)); m_UpstreamApply->Initialize(); diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h index 161e47e06..e00afcd61 100644 --- a/zenserver/compute/apply.h +++ b/zenserver/compute/apply.h @@ -23,6 +23,11 @@ namespace zen { class CasStore; class CidStore; class UpstreamApply; +class CloudCacheClient; +class AuthMgr; + +struct UpstreamAuthConfig; +struct CloudCacheClientOptions; /** * Lambda style compute function service @@ -30,7 +35,14 @@ class UpstreamApply; class HttpFunctionService : public HttpService { public: - HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& SandboxBaseDir); + HttpFunctionService(CasStore& Store, + CidStore& InCidStore, + const std::filesystem::path& BaseDir, + const CloudCacheClientOptions& ComputeOptions, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const UpstreamAuthConfig& StorageAuthConfig, + AuthMgr& Mgr); ~HttpFunctionService(); virtual const char* BaseUri() const override; diff --git a/zenserver/config.cpp b/zenserver/config.cpp index bcacc16c0..adb079d83 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -345,6 +345,62 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"), ""); + options.add_option("compute", + "", + "upstream-horde-url", + "URL to a Horde instance.", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.Url)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-oauth-url", + "URL to the OAuth provier", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthUrl)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-oauth-clientid", + "The OAuth client ID", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientId)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-oauth-clientsecret", + "The OAuth client secret", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientSecret)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-openid-provider", + "Name of a registered Open ID provider", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.OpenIdProvider)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-token", + "A static authentication token", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.AccessToken)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-cluster", + "The Horde compute cluster id", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.Cluster)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-namespace", + "The Jupiter namespace to use with Horde compute", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.Namespace)->default_value(""), + ""); + options.add_option("gc", "", "gc-enabled", @@ -596,6 +652,45 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio } } + if (sol::optional<sol::table> ComputeConfig = lua["compute"]) + { + ServerOptions.ComputeServiceEnabled = ComputeConfig->get_or("enable", ServerOptions.ComputeServiceEnabled); + + if (auto UpstreamConfig = ComputeConfig->get<sol::optional<sol::table>>("upstream")) + { + if (auto HordeConfig = UpstreamConfig->get<sol::optional<sol::table>>("horde")) + { + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("name"), + ServerOptions.UpstreamCacheConfig.HordeConfig.Name); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("url"), + ServerOptions.UpstreamCacheConfig.HordeConfig.Url); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("oauthprovider"), + ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthUrl); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("oauthclientid"), + ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientId); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("oauthclientsecret"), + ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientSecret); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("openidprovider"), + ServerOptions.UpstreamCacheConfig.HordeConfig.OpenIdProvider); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("token"), + ServerOptions.UpstreamCacheConfig.HordeConfig.AccessToken); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("cluster"), + ServerOptions.UpstreamCacheConfig.HordeConfig.Cluster); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("namespace"), + ServerOptions.UpstreamCacheConfig.HordeConfig.Namespace); + }; + } + } + if (sol::optional<sol::table> GcConfig = lua["gc"]) { ServerOptions.GcConfig.IntervalSeconds = GcConfig.value().get_or("intervalseconds", 0); diff --git a/zenserver/config.h b/zenserver/config.h index fd569bdb1..a7a7815a8 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -33,6 +33,19 @@ struct ZenUpstreamJupiterConfig bool UseLegacyDdc = false; }; +struct ZenUpstreamHordeConfig +{ + std::string Name; + std::string Url; + std::string OAuthUrl; + std::string OAuthClientId; + std::string OAuthClientSecret; + std::string OpenIdProvider; + std::string AccessToken; + std::string Cluster; + std::string Namespace; +}; + struct ZenUpstreamZenConfig { std::string Name; @@ -51,6 +64,7 @@ enum class UpstreamCachePolicy : uint8_t struct ZenUpstreamCacheConfig { ZenUpstreamJupiterConfig JupiterConfig; + ZenUpstreamHordeConfig HordeConfig; ZenUpstreamZenConfig ZenConfig; int32_t UpstreamThreadCount = 4; int32_t ConnectTimeoutMilliseconds = 5000; @@ -106,6 +120,7 @@ struct ZenServerOptions bool IsTest = false; bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements bool StructuredCacheEnabled = true; + bool ComputeServiceEnabled = true; bool ShouldCrash = false; // Option for testing crash handling bool IsFirstRun = false; bool NoSentry = false; diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp deleted file mode 100644 index 4bd3ec90a..000000000 --- a/zenserver/experimental/frontend.cpp +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "frontend.h" - -#include <zencore/filesystem.h> -#include <zencore/string.h> - -namespace zen { - -namespace html { - - constexpr std::string_view Index = R"( -<!DOCTYPE html> -<html> -<head> -<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU" crossorigin="anonymous"> -<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.min.js" integrity="sha384-skAcpIdS7UcVUC05LJ9Dxay8AXcDYfBJqt1CJ85S/CFujBsIzCIv+l9liuYLaMQ/" crossorigin="anonymous"></script> -<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css"> -<style type="text/css"> -body { - background-color: #fafafa; -} -</style> -<script type="text/javascript"> - const getCacheStats = () => { - const opts = { headers: { "Accept": "application/json" } }; - fetch("/stats/z$", opts) - .then(response => { - if (!response.ok) { - throw Error(response.statusText); - } - return response.json(); - }) - .then(json => { - document.getElementById("status").innerHTML = "connected" - document.getElementById("stats").innerHTML = JSON.stringify(json, null, 4); - }) - .catch(error => { - document.getElementById("status").innerHTML = "disconnected" - document.getElementById("stats").innerHTML = "" - console.log(error); - }) - .finally(() => { - window.setTimeout(getCacheStats, 1000); - }); - }; - getCacheStats(); -</script> -</head> -<body> - <div class="container"> - <div class="row"> - <div class="text-center mt-5"> - <pre> -__________ _________ __ -\____ / ____ ____ / _____/_/ |_ ____ _______ ____ - / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ - / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/ -/_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ > - \/ \/ \/ \/ \/ - </pre> - <pre id="status"/> - </div> - </div> - <div class="row"> - <pre class="mb-0">Z$:</pre> - <pre id="stats"></pre> - <div> - </div> -</body> -</html> -)"; - -} // namespace html - -HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory) -{ -} - -HttpFrontendService::~HttpFrontendService() -{ -} - -const char* -HttpFrontendService::BaseUri() const -{ - return "/dashboard"; // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp -} - -void -HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) -{ - using namespace std::literals; - - if (m_Directory.empty()) - { - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, html::Index); - } - else - { - std::string_view Uri = Request.RelativeUri(); - std::filesystem::path RelPath{Uri.empty() ? "index.html" : Uri}; - std::filesystem::path AbsPath = m_Directory / RelPath; - - FileContents File = ReadFile(AbsPath); - - if (!File.ErrorCode) - { - // TODO: Map file extension to MIME type - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, File.Data[0]); - } - else - { - return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Ooops!"sv); - } - } -} - -} // namespace zen diff --git a/zenserver/experimental/frontend.h b/zenserver/experimental/frontend.h deleted file mode 100644 index 2ae20e940..000000000 --- a/zenserver/experimental/frontend.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zenhttp/httpserver.h> - -#include <filesystem> - -namespace zen { - -class HttpFrontendService final : public zen::HttpService -{ -public: - HttpFrontendService(std::filesystem::path Directory); - virtual ~HttpFrontendService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; - -private: - std::filesystem::path m_Directory; -}; - -} // namespace zen diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp new file mode 100644 index 000000000..b87d7e313 --- /dev/null +++ b/zenserver/frontend/frontend.cpp @@ -0,0 +1,250 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "frontend.h" + +#include <zencore/endian.h> +#include <zencore/filesystem.h> +#include <zencore/string.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#if ZEN_PLATFORM_WINDOWS +# include <Windows.h> +#endif +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +static IoBuffer FindZipFsInBinary(const IoBuffer& BinBuffer) +{ + if (BinBuffer.GetSize() < 4) + { + return {}; + } + + uintptr_t Cursor = uintptr_t(BinBuffer.GetData()); + size_t BinSize = 0; + + uint32_t Magic = *(uint32_t*)(BinBuffer.GetData()); +#if ZEN_PLATFORM_LINUX + if (Magic == 0x464c457f) + { + struct Elf64Header + { + char Ident[16]; + uint16_t Type; + uint16_t Machine; + uint32_t Version; + uint64_t Entry; + uint64_t ProgHeaderOffset; + uint64_t SectionHeaderOffset; + uint32_t Flags; + uint16_t EhSize; + uint16_t ProgHeaderEntrySize; + uint16_t ProgHeaderCount; + uint16_t SectionHeaderEntrySize; + uint16_t SectionHeaderCount; + uint16_t SectionStringIndex; + }; + + struct SectionHeader + { + uint32_t NameIndex; + uint32_t Type; + uint64_t Flags; + uint64_t Address; + uint64_t Offset; + uint64_t Size; + uint64_t _Other[3]; + }; + + const auto* Elf = (Elf64Header*)Cursor; + if (Elf->Ident[4] != 0x02) // Elf64 + { + return {}; + } + + const auto* Section = (SectionHeader*)(Cursor + Elf->SectionHeaderOffset); + + /* + size_t BinSize = 0; + for (int i = 0, n = Elf->SectionHeaderCount; i < n; ++i, ++Section) + { + uint32_t SectionEnd = Section->Offset + Section->Size; + BinSize = (SectionEnd > BinSize) ? SectionEnd : BinSize; + } + */ + + // What if the section headers aren't the last thing in the fiile though? + BinSize = Elf->SectionHeaderEntrySize; + BinSize *= Elf->SectionHeaderCount; + BinSize += Elf->SectionHeaderOffset; + } +#elif ZEN_PLATFORM_WINDOWS + if ((Magic & 0xffff) == 0x5a4d) + { + const auto* Dos = (IMAGE_DOS_HEADER*)Cursor; + const auto* Nt = (IMAGE_NT_HEADERS64*)(Cursor + Dos->e_lfanew); + const auto* Section = (IMAGE_SECTION_HEADER*)(uintptr_t(&Nt->OptionalHeader) + Nt->FileHeader.SizeOfOptionalHeader); + + for (int i = 0, n = Nt->FileHeader.NumberOfSections; i < n; ++i, ++Section) + { + uint32_t SectionEnd = Section->PointerToRawData + Section->SizeOfRawData; + BinSize = (SectionEnd > BinSize) ? SectionEnd : BinSize; + } + } +#elif ZEN_PLATFORM_MAC + if (Magic == 0xbebafeca) + { + struct MachInt32 + { + operator uint32_t () const { return ByteSwap(Value); } + uint32_t Value; + }; + + struct MachFatArch + { + MachInt32 CpuType; + MachInt32 SubType; + MachInt32 Offset; + MachInt32 Size; + MachInt32 Alignment; + }; + + struct MachFatHeader + { + uint32_t Magic; + MachInt32 NumArchs; + MachFatArch Archs[]; + }; + + const auto* Header = (MachFatHeader*)Cursor; + for (int i = 0, n = Header->NumArchs; i < n; ++i) + { + const MachFatArch* Arch = Header->Archs + i; + uint32_t ArchEnd = Arch->Offset + Arch->Size; + BinSize = (ArchEnd > BinSize) ? ArchEnd : BinSize; + } + } +#endif // win/linux/mac + + if (!BinSize || BinSize > BinBuffer.GetSize()) + { + return {}; + } + + return IoBuffer(BinBuffer, BinSize); +} + +//////////////////////////////////////////////////////////////////////////////// +HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) +: m_Directory(Directory) +{ + std::filesystem::path SelfPath = GetRunningExecutablePath(); + + // Locate a .zip file appended onto the end of this binary + IoBuffer SelfBuffer = IoBufferBuilder::MakeFromFile(SelfPath); + IoBuffer SelfTailBuffer = FindZipFsInBinary(SelfBuffer); + if (SelfTailBuffer) + { + m_ZipFs = ZipFs(std::move(SelfTailBuffer)); + } + +#if ZEN_BUILD_DEBUG + if (!Directory.empty()) + { + return; + } + + std::error_code ErrorCode; + for (auto Path = SelfPath.parent_path(); !Path.empty(); Path = Path.parent_path()) + { + if (!std::filesystem::is_regular_file(Path / "xmake.lua", ErrorCode)) + { + continue; + } + + auto HtmlDir = (Path / __FILE__).parent_path() / "html"; + if (std::filesystem::is_directory(HtmlDir, ErrorCode)) + { + m_Directory = HtmlDir; + } + break; + } +#endif +} + +//////////////////////////////////////////////////////////////////////////////// +HttpFrontendService::~HttpFrontendService() +{ +} + +//////////////////////////////////////////////////////////////////////////////// +const char* +HttpFrontendService::BaseUri() const +{ + return "/dashboard"; // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp +} + +//////////////////////////////////////////////////////////////////////////////// +void +HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) +{ + using namespace std::literals; + + std::string_view Uri = Request.RelativeUri(); + for (; Uri[0] == '/'; Uri = Uri.substr(1)); + if (Uri.empty()) + { + Uri = "index.html"sv; + } + + // Dismiss if the URI contains .. anywhere to prevent arbitrary file reads + if (Uri.find("..") != Uri.npos) + { + Request.WriteResponse(HttpResponseCode::Forbidden); + return; + } + + // Map the file extension to a MIME type. To keep things constrained, only a + // small subset of file extensions is allowed. + HttpContentType ContentType = HttpContentType::kCOUNT; + size_t DotIndex = Uri.rfind("."); + if (DotIndex != Uri.npos) + { + const std::string_view DotExt = Uri.substr(DotIndex); + if (DotExt == ".html") ContentType = HttpContentType::kHTML; + else if (DotExt == ".js") ContentType = HttpContentType::kJSON; + else if (DotExt == ".css") ContentType = HttpContentType::kCSS; + else if (DotExt == ".png") ContentType = HttpContentType::kPNG; + else if (DotExt == ".ico") ContentType = HttpContentType::kIcon; + } + + if (ContentType == HttpContentType::kCOUNT) + { + Request.WriteResponse(HttpResponseCode::Forbidden); + return; + } + + // The given content directory overrides any zip-fs discovered in the binary + if (!m_Directory.empty()) + { + FileContents File = ReadFile(m_Directory / Uri); + if (!File.ErrorCode) + { + Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]); + return; + } + } + + IoBuffer FileBuffer = m_ZipFs.GetFile(Uri); + if (FileBuffer) + { + Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer); + return; + } + + Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); +} + +} // namespace zen diff --git a/zenserver/frontend/frontend.h b/zenserver/frontend/frontend.h new file mode 100644 index 000000000..bf5298169 --- /dev/null +++ b/zenserver/frontend/frontend.h @@ -0,0 +1,25 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenhttp/httpserver.h> +#include "zipfs.h" + +#include <filesystem> + +namespace zen { + +class HttpFrontendService final : public zen::HttpService +{ +public: + HttpFrontendService(std::filesystem::path Directory); + virtual ~HttpFrontendService(); + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + ZipFs m_ZipFs; + std::filesystem::path m_Directory; +}; + +} // namespace zen diff --git a/zenserver/frontend/html/index.html b/zenserver/frontend/html/index.html new file mode 100644 index 000000000..252ee621e --- /dev/null +++ b/zenserver/frontend/html/index.html @@ -0,0 +1,59 @@ +<!DOCTYPE html> +<html> +<head> + <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU" crossorigin="anonymous"> + <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.min.js" integrity="sha384-skAcpIdS7UcVUC05LJ9Dxay8AXcDYfBJqt1CJ85S/CFujBsIzCIv+l9liuYLaMQ/" crossorigin="anonymous"></script> + <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css"> + <style type="text/css"> + body { + background-color: #fafafa; + } + </style> + <script type="text/javascript"> + const getCacheStats = () => { + const opts = { headers: { "Accept": "application/json" } }; + fetch("/stats/z$", opts) + .then(response => { + if (!response.ok) { + throw Error(response.statusText); + } + return response.json(); + }) + .then(json => { + document.getElementById("status").innerHTML = "connected" + document.getElementById("stats").innerHTML = JSON.stringify(json, null, 4); + }) + .catch(error => { + document.getElementById("status").innerHTML = "disconnected" + document.getElementById("stats").innerHTML = "" + console.log(error); + }) + .finally(() => { + window.setTimeout(getCacheStats, 1000); + }); + }; + getCacheStats(); + </script> +</head> +<body> + <div class="container"> + <div class="row"> + <div class="text-center mt-5"> + <pre> +__________ _________ __ +\____ / ____ ____ / _____/_/ |_ ____ _______ ____ + / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ + / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/ +/_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ > + \/ \/ \/ \/ \/ + </pre> + <pre id="status"/> + </div> + </div> + <div class="row"> + <pre class="mb-0">Z$:</pre> + <pre id="stats"></pre> + <div> + </div> +</body> +</html> diff --git a/zenserver/frontend/zipfs.cpp b/zenserver/frontend/zipfs.cpp new file mode 100644 index 000000000..5fb9d0177 --- /dev/null +++ b/zenserver/frontend/zipfs.cpp @@ -0,0 +1,170 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zipfs.h" + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +namespace { + +#if ZEN_COMPILER_MSC +# pragma warning(push) +# pragma warning(disable : 4200) +#endif + +using ZipInt16 = uint16_t; + +struct ZipInt32 +{ + operator uint32_t () const { return *(uint32_t*)Parts; } + uint16_t Parts[2]; +}; + +struct EocdRecord +{ + enum : uint32_t { + Magic = 0x0605'4b50, + }; + ZipInt32 Signature; + ZipInt16 ThisDiskIndex; + ZipInt16 CdStartDiskIndex; + ZipInt16 CdRecordThisDiskCount; + ZipInt16 CdRecordCount; + ZipInt32 CdSize; + ZipInt32 CdOffset; + ZipInt16 CommentSize; + char Comment[]; +}; + +struct CentralDirectoryRecord +{ + enum : uint32_t { + Magic = 0x0201'4b50, + }; + + ZipInt32 Signature; + ZipInt16 VersionMadeBy; + ZipInt16 VersionRequired; + ZipInt16 Flags; + ZipInt16 CompressionMethod; + ZipInt16 LastModTime; + ZipInt16 LastModDate; + ZipInt32 Crc32; + ZipInt32 CompressedSize; + ZipInt32 OriginalSize; + ZipInt16 FileNameLength; + ZipInt16 ExtraFieldLength; + ZipInt16 CommentLength; + ZipInt16 DiskIndex; + ZipInt16 InternalFileAttr; + ZipInt32 ExternalFileAttr; + ZipInt32 Offset; + char FileName[]; +}; + +struct LocalFileHeader +{ + enum : uint32_t { + Magic = 0x0304'4b50, + }; + + ZipInt32 Signature; + ZipInt16 VersionRequired; + ZipInt16 Flags; + ZipInt16 CompressionMethod; + ZipInt16 LastModTime; + ZipInt16 LastModDate; + ZipInt32 Crc32; + ZipInt32 CompressedSize; + ZipInt32 OriginalSize; + ZipInt16 FileNameLength; + ZipInt16 ExtraFieldLength; + char FileName[]; +}; + +#if ZEN_COMPILER_MSC +# pragma warning(pop) +#endif + +} // namespace + + + +////////////////////////////////////////////////////////////////////////// +ZipFs::ZipFs(IoBuffer&& Buffer) +{ + MemoryView View = Buffer.GetView(); + + uint8_t* Cursor = (uint8_t*)(View.GetData()) + View.GetSize(); + if (View.GetSize() < sizeof(EocdRecord)) + { + return; + } + + const auto* EocdCursor = (EocdRecord*)(Cursor - sizeof(EocdRecord)); + + // It is more correct to search backwards for EocdRecord::Magic as the + // comment can be of a variable length. But here we're not going to support + // zip files with comments. + if (EocdCursor->Signature != EocdRecord::Magic) + { + return; + } + + // Zip64 isn't supported either + if (EocdCursor->ThisDiskIndex == 0xffff) + { + return; + } + + Cursor -= View.GetSize(); + + const auto* CdCursor = (CentralDirectoryRecord*)(Cursor + EocdCursor->CdOffset); + for (int i = 0, n = EocdCursor->CdRecordCount; i < n; ++i) + { + const CentralDirectoryRecord& Cd = *CdCursor; + + bool Acceptable = true; + Acceptable &= (Cd.OriginalSize > 0); // has some content + Acceptable &= (Cd.CompressionMethod == 0); // is stored uncomrpessed + if (Acceptable) + { + const uint8_t* Lfh = Cursor + Cd.Offset; + if (uintptr_t(Lfh - Cursor) < View.GetSize()) + { + std::string_view FileName(Cd.FileName, Cd.FileNameLength); + m_Files.insert(std::make_pair(FileName, FileItem{Lfh, size_t(0)})); + } + } + + uint32_t ExtraBytes = Cd.FileNameLength + Cd.ExtraFieldLength + Cd.CommentLength; + CdCursor = (CentralDirectoryRecord*)(Cd.FileName + ExtraBytes); + } + + m_Buffer = std::move(Buffer); +} + +////////////////////////////////////////////////////////////////////////// +IoBuffer ZipFs::GetFile(const std::string_view& FileName) const +{ + FileMap::iterator Iter = m_Files.find(FileName); + if (Iter == m_Files.end()) + { + return{}; + } + + FileItem& Item = Iter->second; + if (Item.GetSize() > 0) + { + return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); + } + + const auto* Lfh = (LocalFileHeader*)(Item.GetData()); + Item = MemoryView( + Lfh->FileName + Lfh->FileNameLength + Lfh->ExtraFieldLength, + Lfh->OriginalSize + ); + return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); +} + +} // namespace zen diff --git a/zenserver/frontend/zipfs.h b/zenserver/frontend/zipfs.h new file mode 100644 index 000000000..a304e9ff5 --- /dev/null +++ b/zenserver/frontend/zipfs.h @@ -0,0 +1,24 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/iobuffer.h> + +#include <unordered_map> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +class ZipFs +{ +public: + ZipFs() = default; + ZipFs(IoBuffer&& Buffer); + IoBuffer GetFile(const std::string_view& FileName) const; + +private: + using FileItem = MemoryView; + using FileMap = std::unordered_map<std::string_view, FileItem>; + FileMap mutable m_Files; + IoBuffer m_Buffer; +}; + +} // namespace zen diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 2b064a610..eef1daab0 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -577,12 +577,12 @@ CloudCacheSession::ObjectExists(const std::set<IoHash>& Keys) } CloudCacheResult -CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksData) +CloudCacheSession::PostComputeTasks(IoBuffer TasksData) { ZEN_TRACE_CPU("HordeClient::PostComputeTasks"); ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << m_CacheClient->ComputeCluster(); cpr::Session& Session = GetSession(); const CloudCacheAccessToken& AccessToken = GetAccessToken(); @@ -609,8 +609,11 @@ CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksDa CloudCacheResult CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t WaitSeconds) { + ZEN_TRACE_CPU("HordeClient::GetComputeUpdates"); + ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId << "/updates?wait=" << WaitSeconds; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << m_CacheClient->ComputeCluster() << "/updates/" << ChannelId + << "?wait=" << WaitSeconds; cpr::Session& Session = GetSession(); const CloudCacheAccessToken& AccessToken = GetAccessToken(); @@ -703,6 +706,8 @@ CloudCacheSession::VerifyAccessToken(long StatusCode) CloudCacheResult CloudCacheSession::CacheTypeExists(std::string_view TypeId, const IoHash& Key) { + ZEN_TRACE_CPU("HordeClient::CacheTypeExists"); + ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/" << TypeId << "/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); @@ -731,6 +736,8 @@ CloudCacheSession::CacheTypeExists(std::string_view TypeId, const IoHash& Key) CloudCacheExistsResult CloudCacheSession::CacheTypeExists(std::string_view TypeId, const std::set<IoHash>& Keys) { + ZEN_TRACE_CPU("HordeClient::CacheTypeExists"); + ExtendableStringBuilder<256> Query; for (const auto& Key : Keys) { @@ -766,7 +773,7 @@ CloudCacheSession::CacheTypeExists(std::string_view TypeId, const std::set<IoHas { IoBuffer Buffer = IoBuffer(zen::IoBuffer::Wrap, Response.text.data(), Response.text.size()); const CbObject ExistsResponse = LoadCompactBinaryObject(Buffer); - for (auto& Item : ExistsResponse["id"sv]) + for (auto& Item : ExistsResponse["Needs"sv]) { if (Item.IsHash()) { @@ -878,6 +885,7 @@ CloudCacheClient::CloudCacheClient(const CloudCacheClientOptions& Options, std:: , m_ServiceUrl(Options.ServiceUrl) , m_DdcNamespace(Options.DdcNamespace) , m_BlobStoreNamespace(Options.BlobStoreNamespace) +, m_ComputeCluster(Options.ComputeCluster) , m_ConnectTimeout(Options.ConnectTimeout) , m_Timeout(Options.Timeout) , m_TokenProvider(std::move(TokenProvider)) diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index ddd7ea160..9854e6f1e 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -116,7 +116,7 @@ public: CloudCacheExistsResult CompressedBlobExists(const std::set<IoHash>& Keys); CloudCacheExistsResult ObjectExists(const std::set<IoHash>& Keys); - CloudCacheResult PostComputeTasks(std::string_view ChannelId, IoBuffer TasksData); + CloudCacheResult PostComputeTasks(IoBuffer TasksData); CloudCacheResult GetComputeUpdates(std::string_view ChannelId, const uint32_t WaitSeconds = 0); CloudCacheResult GetObjectTree(const IoHash& Key); @@ -167,6 +167,7 @@ struct CloudCacheClientOptions std::string_view ServiceUrl; std::string_view DdcNamespace; std::string_view BlobStoreNamespace; + std::string_view ComputeCluster; std::chrono::milliseconds ConnectTimeout{5000}; std::chrono::milliseconds Timeout{}; bool UseLegacyDdc = false; @@ -184,6 +185,7 @@ public: CloudCacheAccessToken AcquireAccessToken(); std::string_view DdcNamespace() const { return m_DdcNamespace; } std::string_view BlobStoreNamespace() const { return m_BlobStoreNamespace; } + std::string_view ComputeCluster() const { return m_ComputeCluster; } std::string_view ServiceUrl() const { return m_ServiceUrl; } spdlog::logger& Logger() { return m_Log; } @@ -193,6 +195,7 @@ private: std::string m_ServiceUrl; std::string m_DdcNamespace; std::string m_BlobStoreNamespace; + std::string m_ComputeCluster; std::chrono::milliseconds m_ConnectTimeout{}; std::chrono::milliseconds m_Timeout{}; std::unique_ptr<CloudCacheTokenProvider> m_TokenProvider; diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index 63c334265..918697224 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -23,6 +23,9 @@ # include <zenstore/cas.h> # include <zenstore/cidstore.h> +# include <auth/authmgr.h> +# include <upstream/upstreamcache.h> + # include "cache/structuredcachestore.h" # include "diag/logging.h" @@ -48,17 +51,76 @@ namespace detail { class HordeUpstreamApplyEndpoint final : public UpstreamApplyEndpoint { public: - HordeUpstreamApplyEndpoint(const CloudCacheClientOptions& Options, - std::unique_ptr<zen::CloudCacheTokenProvider> TokenProvider, - CasStore& CasStore, - CidStore& CidStore) + HordeUpstreamApplyEndpoint(const CloudCacheClientOptions& ComputeOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& StorageAuthConfig, + CasStore& CasStore, + CidStore& CidStore, + AuthMgr& Mgr) : m_Log(logging::Get("upstream-apply")) + , m_AuthMgr(Mgr) , m_CasStore(CasStore) , m_CidStore(CidStore) { - m_DisplayName = fmt::format("Horde - '{}'", Options.ServiceUrl); - m_Client = new CloudCacheClient(Options, std::move(TokenProvider)); + m_DisplayName = fmt::format("{} - '{}'+'{}'", ComputeOptions.Name, ComputeOptions.ServiceUrl, StorageOptions.ServiceUrl); m_ChannelId = fmt::format("zen-{}", zen::GetSessionIdString()); + + { + std::unique_ptr<CloudCacheTokenProvider> TokenProvider; + + if (ComputeAuthConfig.OAuthUrl.empty() == false) + { + TokenProvider = + CloudCacheTokenProvider::CreateFromOAuthClientCredentials({.Url = ComputeAuthConfig.OAuthUrl, + .ClientId = ComputeAuthConfig.OAuthClientId, + .ClientSecret = ComputeAuthConfig.OAuthClientSecret}); + } + else if (ComputeAuthConfig.OpenIdProvider.empty() == false) + { + TokenProvider = + CloudCacheTokenProvider::CreateFromCallback([this, ProviderName = std::string(ComputeAuthConfig.OpenIdProvider)]() { + AuthMgr::OpenIdAccessToken Token = m_AuthMgr.GetOpenIdAccessToken(ProviderName); + return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; + }); + } + else + { + CloudCacheAccessToken AccessToken{.Value = std::string(ComputeAuthConfig.AccessToken), + .ExpireTime = CloudCacheAccessToken::TimePoint::max()}; + TokenProvider = CloudCacheTokenProvider::CreateFromStaticToken(AccessToken); + } + + m_Client = new CloudCacheClient(ComputeOptions, std::move(TokenProvider)); + } + + { + std::unique_ptr<CloudCacheTokenProvider> TokenProvider; + + if (StorageAuthConfig.OAuthUrl.empty() == false) + { + TokenProvider = + CloudCacheTokenProvider::CreateFromOAuthClientCredentials({.Url = StorageAuthConfig.OAuthUrl, + .ClientId = StorageAuthConfig.OAuthClientId, + .ClientSecret = StorageAuthConfig.OAuthClientSecret}); + } + else if (StorageAuthConfig.OpenIdProvider.empty() == false) + { + TokenProvider = + CloudCacheTokenProvider::CreateFromCallback([this, ProviderName = std::string(StorageAuthConfig.OpenIdProvider)]() { + AuthMgr::OpenIdAccessToken Token = m_AuthMgr.GetOpenIdAccessToken(ProviderName); + return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; + }); + } + else + { + CloudCacheAccessToken AccessToken{.Value = std::string(StorageAuthConfig.AccessToken), + .ExpireTime = CloudCacheAccessToken::TimePoint::max()}; + TokenProvider = CloudCacheTokenProvider::CreateFromStaticToken(AccessToken); + } + + m_StorageClient = new CloudCacheClient(StorageOptions, std::move(TokenProvider)); + } } virtual ~HordeUpstreamApplyEndpoint() = default; @@ -109,10 +171,11 @@ namespace detail { m_PendingTasks[UpstreamData.TaskId] = ApplyRecord; } - CloudCacheSession Session(m_Client); + CloudCacheSession ComputeSession(m_Client); + CloudCacheSession StorageSession(m_StorageClient); { - CloudCacheResult Result = BatchPutBlobsIfMissing(Session, UpstreamData.Blobs); + CloudCacheResult Result = BatchPutBlobsIfMissing(StorageSession, UpstreamData.Blobs); Bytes += Result.Bytes; ElapsedSeconds += Result.ElapsedSeconds; if (!Result.Success) @@ -126,7 +189,7 @@ namespace detail { } { - CloudCacheResult Result = BatchPutObjectsIfMissing(Session, UpstreamData.Objects); + CloudCacheResult Result = BatchPutObjectsIfMissing(StorageSession, UpstreamData.Objects); Bytes += Result.Bytes; ElapsedSeconds += Result.ElapsedSeconds; if (!Result.Success) @@ -136,17 +199,33 @@ namespace detail { .Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds}; } + + Result = StorageSession.PutRef("requests"sv, + UpstreamData.TaskId, + UpstreamData.Objects[UpstreamData.TaskId].GetBuffer().AsIoBuffer(), + ZenContentType::kCbObject); + + Bytes += Result.Bytes; + ElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) + { + return {.Error{.ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1, + .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to add task ref"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } UpstreamData.Objects.clear(); } CbObjectWriter Writer; + Writer.AddString("c"sv, m_ChannelId); Writer.AddObjectAttachment("r"sv, UpstreamData.RequirementsId); Writer.BeginArray("t"sv); Writer.AddObjectAttachment(UpstreamData.TaskId); Writer.EndArray(); IoBuffer TasksData = Writer.Save().GetBuffer().AsIoBuffer(); - CloudCacheResult Result = Session.PostComputeTasks(m_ChannelId, TasksData); + CloudCacheResult Result = ComputeSession.PostComputeTasks(TasksData); Bytes += Result.Bytes; ElapsedSeconds += Result.ElapsedSeconds; if (!Result.Success) @@ -214,8 +293,8 @@ namespace detail { { return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, - .ErrorCode = ExistsResult.ErrorCode ? ExistsResult.ErrorCode : -1, - .Reason = !ExistsResult.Reason.empty() ? std::move(ExistsResult.Reason) : "Failed to put blobs"}; + .ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1, + .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to put blobs"}; } } @@ -265,8 +344,8 @@ namespace detail { { return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, - .ErrorCode = ExistsResult.ErrorCode ? ExistsResult.ErrorCode : -1, - .Reason = !ExistsResult.Reason.empty() ? std::move(ExistsResult.Reason) : "Failed to put objects"}; + .ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1, + .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to put objects"}; } } @@ -330,9 +409,10 @@ namespace detail { try { - CloudCacheSession Session(m_Client); + CloudCacheSession ComputeSession(m_Client); + CloudCacheSession StorageSession(m_StorageClient); - CloudCacheResult UpdatesResult = Session.GetComputeUpdates(m_ChannelId); + CloudCacheResult UpdatesResult = ComputeSession.GetComputeUpdates(m_ChannelId); Bytes += UpdatesResult.Bytes; ElapsedSeconds += UpdatesResult.ElapsedSeconds; if (!UpdatesResult.Success) @@ -349,9 +429,6 @@ namespace detail { CbObject TaskStatus = LoadCompactBinaryObject(UpdatesResult.Response); - // zen::StringBuilder<4096> ObjStr; - // zen::CompactBinaryToJson(TaskStatus, ObjStr); - for (auto& It : TaskStatus["u"sv]) { CbObjectView Status = It.AsObjectView(); @@ -366,7 +443,7 @@ namespace detail { continue; } - const IoHash TaskId = Status["h"sv].AsObjectAttachment(); + const IoHash TaskId = Status["h"sv].AsHash(); IoHash WorkerId; IoHash ActionId; @@ -383,7 +460,7 @@ namespace detail { m_PendingTasks.erase(TaskIt); } - GetUpstreamApplyResult Result = ProcessTaskStatus(Status, Session); + GetUpstreamApplyResult Result = ProcessTaskStatus(Status, StorageSession); Bytes += Result.Bytes; ElapsedSeconds += Result.ElapsedSeconds; @@ -411,9 +488,11 @@ namespace detail { CasStore& m_CasStore; CidStore& m_CidStore; + AuthMgr& m_AuthMgr; spdlog::logger& m_Log; std::string m_DisplayName; RefPtr<CloudCacheClient> m_Client; + RefPtr<CloudCacheClient> m_StorageClient; UpstreamApplyEndpointStats m_Stats; std::atomic_bool m_HealthOk{false}; std::string m_ChannelId; @@ -453,9 +532,9 @@ namespace detail { return {.Error{.ErrorCode = -1, .Reason = fmt::format("Task {}", ComputeTaskOutcomeToString(Outcome))}}; } - const IoHash TaskId = TaskStatus["h"sv].AsObjectAttachment(); + const IoHash TaskId = TaskStatus["h"sv].AsHash(); const DateTime Time = TaskStatus["t"sv].AsDateTime(); - const IoHash ResultHash = TaskStatus["r"sv].AsObjectAttachment(); + const IoHash ResultHash = TaskStatus["r"sv].AsHash(); const std::string_view AgentId = TaskStatus["a"sv].AsString(); const std::string_view LeaseId = TaskStatus["l"sv].AsString(); @@ -463,105 +542,96 @@ namespace detail { double ElapsedSeconds{}; // Get Result object and all Object Attachments + Binary Attachment IDs - CloudCacheResult ObjectTreeResult = Session.GetObjectTree(ResultHash); - Bytes += ObjectTreeResult.Bytes; - ElapsedSeconds += ObjectTreeResult.ElapsedSeconds; + CloudCacheResult ObjectRefResult = Session.GetRef("responses"sv, ResultHash, ZenContentType::kCbObject); + Bytes += ObjectRefResult.Bytes; + ElapsedSeconds += ObjectRefResult.ElapsedSeconds; - if (!ObjectTreeResult.Success) + if (!ObjectRefResult.Success) { return {.Error{.ErrorCode = -1, .Reason = "Failed to get result object data"}, .Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds}; } - std::map<IoHash, IoBuffer> TreeObjectData; - std::map<IoHash, IoBuffer> TreeBinaryData; + std::vector<IoHash> ObjectsToIterate; + std::map<IoHash, IoBuffer> ObjectData; + std::map<IoHash, IoBuffer> BinaryData; - MemoryView ResponseView = ObjectTreeResult.Response; - while (ResponseView.GetSize() > 0) - { - CbFieldView Field = CbFieldView(ResponseView.GetData()); - ResponseView += Field.GetSize(); + ObjectData[ResultHash] = ObjectRefResult.Response; + CbObject Object = LoadCompactBinaryObject(ObjectData[ResultHash]); + Object.IterateAttachments([&](CbFieldView Field) { if (Field.IsObjectAttachment()) { - const IoHash Hash = Field.AsObjectAttachment(); - Field = CbFieldView(ResponseView.GetData()); - ResponseView += Field.GetSize(); - if (!Field.IsObject()) // No data + const IoHash AttachmentHash = Field.AsObjectAttachment(); + if (!ObjectData.contains(AttachmentHash)) { - TreeObjectData[Hash] = {}; - continue; + ObjectsToIterate.push_back(AttachmentHash); } - MemoryView FieldView = Field.AsObjectView().GetView(); - - TreeObjectData[Hash] = IoBuffer(IoBuffer::Wrap, FieldView.GetData(), FieldView.GetSize()); } else if (Field.IsBinaryAttachment()) { - const IoHash Hash = Field.AsBinaryAttachment(); - TreeBinaryData[Hash] = {}; - } - else // Unknown type - { + const IoHash AttachmentHash = Field.AsBinaryAttachment(); + BinaryData[AttachmentHash] = {}; } - } + }); - for (auto& It : TreeObjectData) + while (!ObjectsToIterate.empty()) { - if (It.second.GetSize() == 0) + const IoHash Hash = ObjectsToIterate.back(); + ObjectsToIterate.pop_back(); + + CloudCacheResult ObjectResult = Session.GetObject(Hash); + Bytes += ObjectRefResult.Bytes; + ElapsedSeconds += ObjectRefResult.ElapsedSeconds; + if (!ObjectResult.Success) { - CloudCacheResult ObjectResult = Session.GetObject(It.first); - Bytes += ObjectTreeResult.Bytes; - ElapsedSeconds += ObjectTreeResult.ElapsedSeconds; - if (!ObjectTreeResult.Success) + return {.Error{.ErrorCode = ObjectResult.ErrorCode, .Reason = std::move(ObjectResult.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + ObjectData[Hash] = std::move(ObjectResult.Response); + + CbObject IterateObject = LoadCompactBinaryObject(ObjectData[Hash]); + IterateObject.IterateAttachments([&](CbFieldView Field) { + if (Field.IsObjectAttachment()) { - return {.Error{.ErrorCode = ObjectResult.ErrorCode, .Reason = std::move(ObjectResult.Reason)}, - .Bytes = Bytes, - .ElapsedSeconds = ElapsedSeconds}; + const IoHash AttachmentHash = Field.AsObjectAttachment(); + if (!ObjectData.contains(AttachmentHash)) + { + ObjectsToIterate.push_back(AttachmentHash); + } } - - if (!ObjectResult.Success) + else if (Field.IsBinaryAttachment()) { - return {.Error{.ErrorCode = -1, .Reason = "Failed to get result object attachment data"}, - .Bytes = Bytes, - .ElapsedSeconds = ElapsedSeconds}; + const IoHash AttachmentHash = Field.AsBinaryAttachment(); + BinaryData[AttachmentHash] = {}; } - It.second = std::move(ObjectResult.Response); - } + }); } - for (auto& It : TreeBinaryData) + // Batch load all binary data + for (auto& It : BinaryData) { - if (It.second.GetSize() == 0) + CloudCacheResult BlobResult = Session.GetBlob(It.first); + Bytes += ObjectRefResult.Bytes; + ElapsedSeconds += ObjectRefResult.ElapsedSeconds; + if (!BlobResult.Success) { - CloudCacheResult BlobResult = Session.GetBlob(It.first); - Bytes += ObjectTreeResult.Bytes; - ElapsedSeconds += ObjectTreeResult.ElapsedSeconds; - if (!BlobResult.Success) - { - return {.Error{.ErrorCode = BlobResult.ErrorCode, .Reason = std::move(BlobResult.Reason)}, - .Bytes = Bytes, - .ElapsedSeconds = ElapsedSeconds}; - } - - if (!BlobResult.Success) - { - return {.Error{.ErrorCode = -1, .Reason = "Failed to get result binary attachment data"}, - .Bytes = Bytes, - .ElapsedSeconds = ElapsedSeconds}; - } - It.second = std::move(BlobResult.Response); + return {.Error{.ErrorCode = BlobResult.ErrorCode, .Reason = std::move(BlobResult.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; } + It.second = std::move(BlobResult.Response); } - CbObject ResultObject = LoadCompactBinaryObject(TreeObjectData[ResultHash]); + CbObject ResultObject = LoadCompactBinaryObject(ObjectData[ResultHash]); int32_t ExitCode = ResultObject["e"sv].AsInt32(); IoHash StdOutHash = ResultObject["so"sv].AsBinaryAttachment(); IoHash StdErrHash = ResultObject["se"sv].AsBinaryAttachment(); IoHash OutputHash = ResultObject["o"sv].AsObjectAttachment(); - std::string StdOut = std::string((const char*)TreeBinaryData[StdOutHash].GetData(), TreeBinaryData[StdOutHash].GetSize()); - std::string StdErr = std::string((const char*)TreeBinaryData[StdErrHash].GetData(), TreeBinaryData[StdErrHash].GetSize()); + std::string StdOut = std::string((const char*)BinaryData[StdOutHash].GetData(), BinaryData[StdOutHash].GetSize()); + std::string StdErr = std::string((const char*)BinaryData[StdErrHash].GetData(), BinaryData[StdErrHash].GetSize()); if (ExitCode != 0) { @@ -572,7 +642,7 @@ namespace detail { .StdErr = std::move(StdErr)}; } - CbObject OutputObject = LoadCompactBinaryObject(TreeObjectData[OutputHash]); + CbObject OutputObject = LoadCompactBinaryObject(ObjectData[OutputHash]); // Get build.output IoHash BuildOutputId; @@ -583,7 +653,7 @@ namespace detail { if (FileObject["n"sv].AsString() == "Build.output"sv) { BuildOutputId = FileObject["h"sv].AsBinaryAttachment(); - BuildOutput = TreeBinaryData[BuildOutputId]; + BuildOutput = BinaryData[BuildOutputId]; break; } } @@ -604,7 +674,7 @@ namespace detail { const CbObjectView DirectoryObject = It.AsObjectView(); if (DirectoryObject["n"sv].AsString() == "Outputs"sv) { - OutputDirectoryTree = TreeObjectData[DirectoryObject["h"sv].AsObjectAttachment()]; + OutputDirectoryTree = ObjectData[DirectoryObject["h"sv].AsObjectAttachment()]; break; } } @@ -636,7 +706,7 @@ namespace detail { // Hash is the compressed data hash, and how it is stored in Horde IoHash CompressedId = FileObject["h"sv].AsBinaryAttachment(); - if (!TreeBinaryData.contains(CompressedId)) + if (!BinaryData.contains(CompressedId)) { Log().warn("Object attachment chunk not retrieved from Horde {}", CompressedId.ToHexString()); return {.Error{.ErrorCode = -1, .Reason = "Object attachment chunk not retrieved from Horde"}, @@ -659,7 +729,7 @@ namespace detail { } const IoHash& CompressedId = CidToCompressedId.at(DecompressedId); - if (!TreeBinaryData.contains(CompressedId)) + if (!BinaryData.contains(CompressedId)) { Log().warn("Missing output {} compressed {} uncompressed", CompressedId.ToHexString(), @@ -668,7 +738,7 @@ namespace detail { return; } - CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(TreeBinaryData[CompressedId])); + CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(BinaryData[CompressedId])); if (!AttachmentBuffer) { @@ -826,11 +896,11 @@ namespace detail { int64_t Memory = ApplyRecord.WorkerDescriptor["memory"sv].AsInt64(); bool Exclusive = ApplyRecord.WorkerDescriptor["exclusive"sv].AsBool(); - // TODO: Remove override when Horde accepts the UE style Host Platforms (Win64, Linux, Mac) std::string Condition = fmt::format("Platform == '{}'", HostPlatform); if (HostPlatform == "Win64") { - Condition += " && Pool == 'Win-RemoteExec'"; + // TODO + // Condition += " && Pool == 'Win-RemoteExec'"; } std::map<std::string_view, int64_t> Resources; @@ -1176,10 +1246,10 @@ struct UpstreamApplyStats ////////////////////////////////////////////////////////////////////////// -class DefaultUpstreamApply final : public UpstreamApply +class UpstreamApplyImpl final : public UpstreamApply { public: - DefaultUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) + UpstreamApplyImpl(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) : m_Log(logging::Get("upstream-apply")) , m_Options(Options) , m_CasStore(CasStore) @@ -1188,7 +1258,7 @@ public: { } - virtual ~DefaultUpstreamApply() { Shutdown(); } + virtual ~UpstreamApplyImpl() { Shutdown(); } virtual bool Initialize() override { @@ -1213,12 +1283,12 @@ public: for (uint32_t Idx = 0; Idx < m_Options.ThreadCount; Idx++) { - m_UpstreamThreads.emplace_back(&DefaultUpstreamApply::ProcessUpstreamQueue, this); + m_UpstreamThreads.emplace_back(&UpstreamApplyImpl::ProcessUpstreamQueue, this); } - m_UpstreamUpdatesThread = std::thread(&DefaultUpstreamApply::ProcessUpstreamUpdates, this); + m_UpstreamUpdatesThread = std::thread(&UpstreamApplyImpl::ProcessUpstreamUpdates, this); - m_EndpointMonitorThread = std::thread(&DefaultUpstreamApply::MonitorEndpoints, this); + m_EndpointMonitorThread = std::thread(&UpstreamApplyImpl::MonitorEndpoints, this); } return m_RunState.IsRunning; @@ -1558,18 +1628,27 @@ private: ////////////////////////////////////////////////////////////////////////// std::unique_ptr<UpstreamApply> -MakeUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) +UpstreamApply::Create(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) { - return std::make_unique<DefaultUpstreamApply>(Options, CasStore, CidStore); + return std::make_unique<UpstreamApplyImpl>(Options, CasStore, CidStore); } std::unique_ptr<UpstreamApplyEndpoint> -MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, - std::unique_ptr<CloudCacheTokenProvider> TokenProvider, - CasStore& CasStore, - CidStore& CidStore) +UpstreamApplyEndpoint::CreateHordeEndpoint(const CloudCacheClientOptions& ComputeOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& StorageAuthConfig, + CasStore& CasStore, + CidStore& CidStore, + AuthMgr& Mgr) { - return std::make_unique<detail::HordeUpstreamApplyEndpoint>(Options, std::move(TokenProvider), CasStore, CidStore); + return std::make_unique<detail::HordeUpstreamApplyEndpoint>(ComputeOptions, + ComputeAuthConfig, + StorageOptions, + StorageAuthConfig, + CasStore, + CidStore, + Mgr); } } // namespace zen diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h index c56a22ac3..44c08e30e 100644 --- a/zenserver/upstream/upstreamapply.h +++ b/zenserver/upstream/upstreamapply.h @@ -25,6 +25,8 @@ class CidStore; class ZenCacheStore; struct CloudCacheClientOptions; class CloudCacheTokenProvider; +struct UpstreamAuthConfig; +class AuthMgr; enum class UpstreamApplyState : int32_t { @@ -129,10 +131,18 @@ public: virtual PostUpstreamApplyResult PostApply(const UpstreamApplyRecord& ApplyRecord) = 0; virtual GetUpstreamApplyUpdatesResult GetUpdates() = 0; virtual UpstreamApplyEndpointStats& Stats() = 0; + + static std::unique_ptr<UpstreamApplyEndpoint> CreateHordeEndpoint(const CloudCacheClientOptions& ComputeOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& StorageAuthConfig, + CasStore& CasStore, + CidStore& CidStore, + AuthMgr& Mgr); }; /** - * Manages one or more upstream cache endpoints. + * Manages one or more upstream compute endpoints. */ class UpstreamApply { @@ -157,14 +167,9 @@ public: virtual EnqueueResult EnqueueUpstream(UpstreamApplyRecord ApplyRecord) = 0; virtual StatusResult GetStatus(const IoHash& WorkerId, const IoHash& ActionId) = 0; virtual void GetStatus(CbObjectWriter& CbO) = 0; -}; -std::unique_ptr<UpstreamApply> MakeUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore); - -std::unique_ptr<UpstreamApplyEndpoint> MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, - std::unique_ptr<CloudCacheTokenProvider> TokenProvider, - CasStore& CasStore, - CidStore& CidStore); + static std::unique_ptr<UpstreamApply> Create(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore); +}; } // namespace zen diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index ad4f67911..569e3c150 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -17,7 +17,7 @@ target("zenserver") add_ldflags("/MANIFEST:EMBED") add_ldflags("/LTCG") else - del_files("windows/**") + remove_files("windows/**") end if is_plat("macosx") then diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 3c7f9004d..110800315 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -106,7 +106,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include "cache/structuredcachestore.h" #include "compute/apply.h" #include "diag/diagsvcs.h" -#include "experimental/frontend.h" +#include "frontend/frontend.h" #include "experimental/usnjournal.h" #include "monitoring/httpstats.h" #include "monitoring/httpstatus.h" @@ -279,15 +279,20 @@ public: #endif #if ZEN_WITH_COMPUTE_SERVICES - ZEN_INFO("instantiating compute services"); + if (ServerOptions.ComputeServiceEnabled) + { + ZEN_INFO("instantiating compute services"); - std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; - zen::CreateDirectories(SandboxDir); - m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore, SandboxDir); + std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; + zen::CreateDirectories(SandboxDir); + m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore, SandboxDir); - std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply"; - zen::CreateDirectories(ApplySandboxDir); - m_HttpFunctionService = std::make_unique<zen::HttpFunctionService>(*m_CasStore, *m_CidStore, ApplySandboxDir); + InitializeCompute(ServerOptions); + } + else + { + ZEN_INFO("NOT instantiating compute services"); + } #endif // ZEN_WITH_COMPUTE_SERVICES if (ServerOptions.StructuredCacheEnabled) @@ -327,14 +332,14 @@ public: m_Http->RegisterService(m_CasService); #if ZEN_WITH_COMPUTE_SERVICES - if (m_HttpLaunchService) + if (ServerOptions.ComputeServiceEnabled) { m_Http->RegisterService(*m_HttpLaunchService); - } - if (m_HttpFunctionService) - { - m_Http->RegisterService(*m_HttpFunctionService); + if (m_HttpFunctionService != nullptr) + { + m_Http->RegisterService(*m_HttpFunctionService); + } } #endif // ZEN_WITH_COMPUTE_SERVICES @@ -360,6 +365,9 @@ public: void InitializeState(const ZenServerOptions& ServerOptions); void InitializeStructuredCache(const ZenServerOptions& ServerOptions); +#if ZEN_WITH_COMPUTE_SERVICES + void InitializeCompute(const ZenServerOptions& ServerOptions); +#endif #if ZEN_ENABLE_MESH void StartMesh(int BasePort) @@ -826,6 +834,59 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) } } +#if ZEN_WITH_COMPUTE_SERVICES +void +ZenServer::InitializeCompute(const ZenServerOptions& ServerOptions) +{ + ServerOptions; + const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; + + // Horde compute upstream + if (UpstreamConfig.HordeConfig.Url.empty() == false && UpstreamConfig.HordeConfig.Url.empty() == false) + { + std::string_view EndpointName = UpstreamConfig.HordeConfig.Name.empty() ? "Horde"sv : UpstreamConfig.HordeConfig.Name; + + auto ComputeOptions = + zen::CloudCacheClientOptions{.Name = EndpointName, + .ServiceUrl = UpstreamConfig.HordeConfig.Url, + .ComputeCluster = UpstreamConfig.HordeConfig.Cluster, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds), + .UseLegacyDdc = false}; + + auto ComputeAuthConfig = zen::UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.HordeConfig.OAuthUrl, + .OAuthClientId = UpstreamConfig.HordeConfig.OAuthClientId, + .OAuthClientSecret = UpstreamConfig.HordeConfig.OAuthClientSecret, + .OpenIdProvider = UpstreamConfig.HordeConfig.OpenIdProvider, + .AccessToken = UpstreamConfig.HordeConfig.AccessToken}; + + auto StorageOptions = + zen::CloudCacheClientOptions{.Name = EndpointName, + .ServiceUrl = UpstreamConfig.JupiterConfig.Url, + .BlobStoreNamespace = UpstreamConfig.HordeConfig.Namespace, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}; + + auto StorageAuthConfig = zen::UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.JupiterConfig.OAuthUrl, + .OAuthClientId = UpstreamConfig.JupiterConfig.OAuthClientId, + .OAuthClientSecret = UpstreamConfig.JupiterConfig.OAuthClientSecret, + .OpenIdProvider = UpstreamConfig.JupiterConfig.OpenIdProvider, + .AccessToken = UpstreamConfig.JupiterConfig.AccessToken}; + + std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply"; + zen::CreateDirectories(ApplySandboxDir); + m_HttpFunctionService = std::make_unique<zen::HttpFunctionService>(*m_CasStore, + *m_CidStore, + ApplySandboxDir, + ComputeOptions, + StorageOptions, + ComputeAuthConfig, + StorageAuthConfig, + *m_AuthMgr); + } +} +#endif + //////////////////////////////////////////////////////////////////////////////// class ZenEntryPoint @@ -845,19 +906,75 @@ ZenEntryPoint::ZenEntryPoint(ZenServerOptions& ServerOptions) : m_ServerOptions( { } +#if ZEN_USE_SENTRY +static void +SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) +{ + char LogMessageBuffer[160]; + std::string LogMessage; + const char* MessagePtr = LogMessageBuffer; + + int n = vsnprintf(LogMessageBuffer, sizeof LogMessageBuffer, Message, Args); + + if (n >= int(sizeof LogMessageBuffer)) + { + LogMessage.resize(n + 1); + + n = vsnprintf(LogMessage.data(), LogMessage.size(), Message, Args); + + MessagePtr = LogMessage.c_str(); + } + + switch (Level) + { + case SENTRY_LEVEL_DEBUG: + ConsoleLog().debug("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_INFO: + ConsoleLog().info("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_WARNING: + ConsoleLog().warn("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_ERROR: + ConsoleLog().error("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_FATAL: + ConsoleLog().critical("sentry: {}", MessagePtr); + break; + } +} +#endif + int ZenEntryPoint::Run() { #if ZEN_USE_SENTRY + std::string SentryDatabasePath = PathToUtf8(m_ServerOptions.DataDir / ".sentry-native"); + if (m_ServerOptions.NoSentry == false) { sentry_options_t* SentryOptions = sentry_options_new(); sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284"); - sentry_options_set_database_path(SentryOptions, PathToUtf8(m_ServerOptions.DataDir / ".sentry-native").c_str()); - sentry_init(SentryOptions); + sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); + sentry_options_set_logger(SentryOptions, SentryLogFunction, this); + // sentry_options_set_debug(SentryOptions, 1); - auto _ = zen::MakeGuard([] { sentry_close(); }); + if (int ErrorCode = sentry_init(SentryOptions); ErrorCode == 0) + { + printf("sentry initialized"); + } + else + { + printf("sentry_init returned failure!"); + } } + + auto _ = zen::MakeGuard([] { sentry_close(); }); #endif auto& ServerOptions = m_ServerOptions; |