From 88b826a248ccaa4fdf911b7e1fe8ca2db6538849 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 23 Feb 2022 09:14:49 +0100 Subject: Added a summary about Linux's zenserver AppImage bundle --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index f26f23150..5f5dd4ddb 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,23 @@ Note that the command above to set the build variant to debug is optional. Tests are only built in debug.The `xmake` flags `-vD` can be useful to diagnose `xmake` issues. +### Distribution compatibility + +Builds for Linux have a dependency on a modern version of the C++ library that +supports Zen's use of C++20 and the GCC-11 toolchain used (assuming the above +build instructions are adhered to). However there is no guarantee that the this +dependency is met when building on one Linux install and running on another. For +example, at the time of writing the LTS version of Ubuntu and Debian do not have +GCC-11's version of libstdc++.so.6 available in their package repositories. + +To solve this, `xmake bundle` will bundle the required C++ .so and zenserver +binary together using AppImage. This can briefly be summarised as create a +squashfs file system containing zenserver and any C++ shared objects list with +`lld`. This file system is then concatenated with AppImage's runtime. This is +implemented in `scripts/bundle_linux.sh`. More details can be found here; + +https://github.com/AppImage/AppImageKit + ## Building on Mac Building on Mac is very similar to Linux; install xmake, clone vcpkg and Zen and -- cgit v1.2.3 From d818ddfcd24ab0dd3392e37cc1422ff2efcb2e8f Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 23 Feb 2022 09:15:32 +0100 Subject: remote_build: convert given key file to Unix line endings --- scripts/remote_build.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/remote_build.py b/scripts/remote_build.py index 08e44c9ad..70f9cf9bf 100644 --- a/scripts/remote_build.py +++ b/scripts/remote_build.py @@ -106,13 +106,15 @@ def _local(args): _header("Validating remote host and credentials") - # Validate key file. OpenSSL needs a trailing EOL, LibreSSL doesn't + # Validate key file. Git's SSH uses OpenSSL which needs UNIX line-endings if args.keyfile: with open(args.keyfile, "rt") as key_file: - lines = [x for x in key_file] - if not lines[-1].endswith("\n"): - print("!! ERROR: key file must end with a new line") - return 1 + lines = [x.strip() for x in key_file] + + with open(args.keyfile, "wb") as key_file: + for line in lines: + key_file.write(line.encode() + b"\n") + identity = ("-i", args.keyfile) else: identity = () -- cgit v1.2.3 From 58de5638631d00030f0c02928a1ed065d45cb62f Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 2 Mar 2022 10:50:56 +0100 Subject: xmake's del_files() has been deprecated --- zenserver/xmake.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index ad4f67911..569e3c150 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -17,7 +17,7 @@ target("zenserver") add_ldflags("/MANIFEST:EMBED") add_ldflags("/LTCG") else - del_files("windows/**") + remove_files("windows/**") end if is_plat("macosx") then -- cgit v1.2.3 From dcefd112056b8e7ce7b399f18652d026aef5a559 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 3 Mar 2022 13:10:34 +0100 Subject: Fixed "taking address of constant" compile error --- zencore/compactbinary.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index 902ec26c8..30b0a4dcb 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -1799,7 +1799,7 @@ TEST_CASE("uson") CHECK(NoneField.HasName() == true); CHECK(NoneField.HasValue() == false); CHECK(NoneField.GetHash() == IoHash::HashBuffer(NoneBytes, sizeof NoneBytes)); - CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1)); + CHECK(NoneField.GetView() == MemoryView(NoneBytes + 1, sizeof NoneBytes - 1)); MemoryView SerializedView; CHECK(NoneField.TryGetSerializedView(SerializedView) == false); @@ -1818,7 +1818,7 @@ TEST_CASE("uson") CHECK(NoneField.HasName() == true); CHECK(NoneField.HasValue() == false); CHECK(NoneField.GetHash() == IoHash::HashBuffer(NoneBytes, sizeof NoneBytes)); - CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1)); + CHECK(NoneField.GetView() == MemoryView(NoneBytes + 1, sizeof NoneBytes - 1)); MemoryView SerializedView; CHECK(NoneField.TryGetSerializedView(SerializedView) == false); } -- cgit v1.2.3 From 14b0b7ffa85efe1e66a5e30a979fefee67876e68 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 7 Mar 2022 20:59:41 +0100 Subject: fixed sentry initialization, added sentry logging support the initialization code was refactored at one point to add a NoSentry option, but unfortunately this had a bug which caused sentry to immediately get cleaned up before leaving the if scope --- zenserver/zenserver.cpp | 64 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 78a62e202..243a2bfc1 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -840,19 +840,75 @@ ZenEntryPoint::ZenEntryPoint(ZenServerOptions& ServerOptions) : m_ServerOptions( { } +#if ZEN_USE_SENTRY +void +SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) +{ + char LogMessageBuffer[160]; + std::string LogMessage; + const char* MessagePtr = LogMessageBuffer; + + int n = vsnprintf(LogMessageBuffer, sizeof LogMessageBuffer, Message, Args); + + if (n >= sizeof LogMessageBuffer) + { + LogMessage.resize(n + 1); + + n = vsnprintf(LogMessage.data(), LogMessage.size(), Message, Args); + + MessagePtr = LogMessage.c_str(); + } + + switch (Level) + { + case SENTRY_LEVEL_DEBUG: + ConsoleLog().debug("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_INFO: + ConsoleLog().info("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_WARNING: + ConsoleLog().warn("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_ERROR: + ConsoleLog().error("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_FATAL: + ConsoleLog().critical("sentry: {}", MessagePtr); + break; + } +} +#endif + int ZenEntryPoint::Run() { #if ZEN_USE_SENTRY + std::string SentryDatabasePath = PathToUtf8(m_ServerOptions.DataDir / ".sentry-native"); + if (m_ServerOptions.NoSentry == false) { sentry_options_t* SentryOptions = sentry_options_new(); sentry_options_set_dsn(SentryOptions, "https://8ba3441bebc941c1ae24b8cd2fd25d55@o10593.ingest.sentry.io/5919284"); - sentry_options_set_database_path(SentryOptions, PathToUtf8(m_ServerOptions.DataDir / ".sentry-native").c_str()); - sentry_init(SentryOptions); - - auto _ = zen::MakeGuard([] { sentry_close(); }); + sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); + sentry_options_set_logger(SentryOptions, SentryLogFunction, this); + //sentry_options_set_debug(SentryOptions, 1); + + if (int ErrorCode = sentry_init(SentryOptions); ErrorCode == 0) + { + printf("sentry initialized"); + } + else + { + printf("sentry_init returned failure!"); + } } + + auto _ = zen::MakeGuard([] { sentry_close(); }); #endif auto& ServerOptions = m_ServerOptions; -- cgit v1.2.3 From 2ba53e94d5f3f4afe97ee1fb04e5a0f497e81ae1 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 8 Mar 2022 10:29:01 +0100 Subject: clang-format fixes --- zencore/include/zencore/logging.h | 2 +- zencore/thread.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h index 74ab0f81f..7c2deeed7 100644 --- a/zencore/include/zencore/logging.h +++ b/zencore/include/zencore/logging.h @@ -54,7 +54,7 @@ struct LogCategory #define ZEN_DEFINE_LOG_CATEGORY_STATIC(Category, Name) \ static struct LogCategory##Category : public LogCategory \ { \ - LogCategory##Category() : LogCategory(Name) {} \ + LogCategory##Category() : LogCategory(Name) {} \ } Category; #define ZEN_LOG_TRACE(Category, fmtstr, ...) \ diff --git a/zencore/thread.cpp b/zencore/thread.cpp index 7b1396f5f..c2ecc8d72 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -75,7 +75,7 @@ SetNameInternal(DWORD thread_id, const char* name) #endif #if ZEN_PLATFORM_LINUX -const bool bNoZombieChildren = [] () { +const bool bNoZombieChildren = []() { // When a child process exits it is put into a zombie state until the parent // collects its result. This doesn't fit the Windows-like model that Zen uses // where there is a less strict familial model and no zombification. Ignoring @@ -87,7 +87,7 @@ const bool bNoZombieChildren = [] () { Action.sa_handler = SIG_IGN; sigaction(SIGCHLD, &Action, nullptr); return true; -} (); +}(); #endif void @@ -647,10 +647,10 @@ ProcessHandle::Wait(int TimeoutMs) timespec SleepTime = {0, SleepMs * 1000 * 1000}; for (int i = 0;; i += SleepMs) { -#if ZEN_PLATFORM_MAC +# if ZEN_PLATFORM_MAC int WaitState = 0; waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED); -#endif +# endif if (kill(m_Pid, 0) < 0) { -- cgit v1.2.3 From c76fb7ccc1034d811e5e6769b3fde5d37966a8b4 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 8 Mar 2022 10:29:29 +0100 Subject: Fixed unsigned comparison warning --- zenserver/zenserver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 243a2bfc1..cc681e9a8 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -841,7 +841,7 @@ ZenEntryPoint::ZenEntryPoint(ZenServerOptions& ServerOptions) : m_ServerOptions( } #if ZEN_USE_SENTRY -void +static void SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) { char LogMessageBuffer[160]; @@ -850,7 +850,7 @@ SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[may int n = vsnprintf(LogMessageBuffer, sizeof LogMessageBuffer, Message, Args); - if (n >= sizeof LogMessageBuffer) + if (n >= int(sizeof LogMessageBuffer)) { LogMessage.resize(n + 1); -- cgit v1.2.3 From edcdc0d9bf7b7deb54df3064a2b46fffaf3d8f9f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 9 Mar 2022 11:05:06 +0100 Subject: Remove constexpr in NoneBytes declaration in tests Visual Studio 17.1.1 has a regression causing compilation errors --- zencore/compactbinary.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index 30b0a4dcb..86c9f26da 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -1754,7 +1754,7 @@ TEST_CASE("uson") SUBCASE("CbField(None|Type|Name)") { constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; + const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; CbFieldView NoneField(NoneBytes); CHECK(NoneField.GetSize() == sizeof(NoneBytes)); @@ -1775,7 +1775,7 @@ TEST_CASE("uson") SUBCASE("CbField(None|Type)") { constexpr CbFieldType FieldType = CbFieldType::None; - constexpr const char NoneBytes[] = {char(FieldType)}; + const char NoneBytes[] = {char(FieldType)}; CbFieldView NoneField(NoneBytes); CHECK(NoneField.GetSize() == sizeof NoneBytes); @@ -1792,7 +1792,7 @@ TEST_CASE("uson") SUBCASE("CbField(None|Name)") { constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; + const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; CbFieldView NoneField(NoneBytes + 1, FieldType); CHECK(NoneField.GetSize() == uint64_t(sizeof NoneBytes)); CHECK(NoneField.GetName().compare("Name") == 0); @@ -1811,7 +1811,7 @@ TEST_CASE("uson") SUBCASE("CbField(None|EmptyName)") { constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - constexpr const uint8_t NoneBytes[] = {uint8_t(FieldType), 0}; + const uint8_t NoneBytes[] = {uint8_t(FieldType), 0}; CbFieldView NoneField(NoneBytes + 1, FieldType); CHECK(NoneField.GetSize() == sizeof NoneBytes); CHECK(NoneField.GetName().empty() == true); -- cgit v1.2.3 From 48708ac6bd71ba990d36370a6dfbd2e92197bf6a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 9 Mar 2022 11:05:23 +0100 Subject: clang format --- zencore/compactbinary.cpp | 12 ++++++------ zenserver/zenserver.cpp | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index 86c9f26da..7cc6db68a 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -1754,7 +1754,7 @@ TEST_CASE("uson") SUBCASE("CbField(None|Type|Name)") { constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; + const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; CbFieldView NoneField(NoneBytes); CHECK(NoneField.GetSize() == sizeof(NoneBytes)); @@ -1775,7 +1775,7 @@ TEST_CASE("uson") SUBCASE("CbField(None|Type)") { constexpr CbFieldType FieldType = CbFieldType::None; - const char NoneBytes[] = {char(FieldType)}; + const char NoneBytes[] = {char(FieldType)}; CbFieldView NoneField(NoneBytes); CHECK(NoneField.GetSize() == sizeof NoneBytes); @@ -1792,7 +1792,7 @@ TEST_CASE("uson") SUBCASE("CbField(None|Name)") { constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; + const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'}; CbFieldView NoneField(NoneBytes + 1, FieldType); CHECK(NoneField.GetSize() == uint64_t(sizeof NoneBytes)); CHECK(NoneField.GetName().compare("Name") == 0); @@ -1810,9 +1810,9 @@ TEST_CASE("uson") SUBCASE("CbField(None|EmptyName)") { - constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; - const uint8_t NoneBytes[] = {uint8_t(FieldType), 0}; - CbFieldView NoneField(NoneBytes + 1, FieldType); + constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName; + const uint8_t NoneBytes[] = {uint8_t(FieldType), 0}; + CbFieldView NoneField(NoneBytes + 1, FieldType); CHECK(NoneField.GetSize() == sizeof NoneBytes); CHECK(NoneField.GetName().empty() == true); CHECK(NoneField.HasName() == true); diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index cc681e9a8..576c88cb8 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -848,7 +848,7 @@ SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[may std::string LogMessage; const char* MessagePtr = LogMessageBuffer; - int n = vsnprintf(LogMessageBuffer, sizeof LogMessageBuffer, Message, Args); + int n = vsnprintf(LogMessageBuffer, sizeof LogMessageBuffer, Message, Args); if (n >= int(sizeof LogMessageBuffer)) { @@ -896,8 +896,8 @@ ZenEntryPoint::Run() sentry_options_set_dsn(SentryOptions, "https://8ba3441bebc941c1ae24b8cd2fd25d55@o10593.ingest.sentry.io/5919284"); sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); sentry_options_set_logger(SentryOptions, SentryLogFunction, this); - //sentry_options_set_debug(SentryOptions, 1); - + // sentry_options_set_debug(SentryOptions, 1); + if (int ErrorCode = sentry_init(SentryOptions); ErrorCode == 0) { printf("sentry initialized"); -- cgit v1.2.3 From 0f5f1f9cdcdb03901f6589cf98a78d387b2f7b54 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Tue, 1 Mar 2022 13:11:33 +0100 Subject: Moved experimental/frontend.* to frontend/frontend.* --- zenserver/experimental/frontend.cpp | 119 ------------------------------------ zenserver/experimental/frontend.h | 24 -------- zenserver/frontend/frontend.cpp | 119 ++++++++++++++++++++++++++++++++++++ zenserver/frontend/frontend.h | 24 ++++++++ zenserver/zenserver.cpp | 2 +- 5 files changed, 144 insertions(+), 144 deletions(-) delete mode 100644 zenserver/experimental/frontend.cpp delete mode 100644 zenserver/experimental/frontend.h create mode 100644 zenserver/frontend/frontend.cpp create mode 100644 zenserver/frontend/frontend.h diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp deleted file mode 100644 index 4bd3ec90a..000000000 --- a/zenserver/experimental/frontend.cpp +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "frontend.h" - -#include -#include - -namespace zen { - -namespace html { - - constexpr std::string_view Index = R"( - - - - - - - - - - -
-
-
-
-__________                  _________  __                           
-\____    /  ____    ____   /   _____/_/  |_   ____  _______   ____  
-  /     / _/ __ \  /    \  \_____  \ \   __\ /  _ \ \_  __ \_/ __ \
- /     /_ \  ___/ |   |  \ /        \ |  |  (  <_> ) |  | \/\  ___/ 
-/_______ \ \___  >|___|  //_______  / |__|   \____/  |__|    \___  >
-        \/     \/      \/         \/                             \/ 
-				
-
-			
-
-
-
Z$:
-

-		
-
- - -)"; - -} // namespace html - -HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory) -{ -} - -HttpFrontendService::~HttpFrontendService() -{ -} - -const char* -HttpFrontendService::BaseUri() const -{ - return "/dashboard"; // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp -} - -void -HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) -{ - using namespace std::literals; - - if (m_Directory.empty()) - { - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, html::Index); - } - else - { - std::string_view Uri = Request.RelativeUri(); - std::filesystem::path RelPath{Uri.empty() ? "index.html" : Uri}; - std::filesystem::path AbsPath = m_Directory / RelPath; - - FileContents File = ReadFile(AbsPath); - - if (!File.ErrorCode) - { - // TODO: Map file extension to MIME type - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, File.Data[0]); - } - else - { - return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Ooops!"sv); - } - } -} - -} // namespace zen diff --git a/zenserver/experimental/frontend.h b/zenserver/experimental/frontend.h deleted file mode 100644 index 2ae20e940..000000000 --- a/zenserver/experimental/frontend.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include - -#include - -namespace zen { - -class HttpFrontendService final : public zen::HttpService -{ -public: - HttpFrontendService(std::filesystem::path Directory); - virtual ~HttpFrontendService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; - -private: - std::filesystem::path m_Directory; -}; - -} // namespace zen diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp new file mode 100644 index 000000000..4bd3ec90a --- /dev/null +++ b/zenserver/frontend/frontend.cpp @@ -0,0 +1,119 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "frontend.h" + +#include +#include + +namespace zen { + +namespace html { + + constexpr std::string_view Index = R"( + + + + + + + + + + +
+
+
+
+__________                  _________  __                           
+\____    /  ____    ____   /   _____/_/  |_   ____  _______   ____  
+  /     / _/ __ \  /    \  \_____  \ \   __\ /  _ \ \_  __ \_/ __ \
+ /     /_ \  ___/ |   |  \ /        \ |  |  (  <_> ) |  | \/\  ___/ 
+/_______ \ \___  >|___|  //_______  / |__|   \____/  |__|    \___  >
+        \/     \/      \/         \/                             \/ 
+				
+
+			
+
+
+
Z$:
+

+		
+
+ + +)"; + +} // namespace html + +HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory) +{ +} + +HttpFrontendService::~HttpFrontendService() +{ +} + +const char* +HttpFrontendService::BaseUri() const +{ + return "/dashboard"; // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp +} + +void +HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) +{ + using namespace std::literals; + + if (m_Directory.empty()) + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, html::Index); + } + else + { + std::string_view Uri = Request.RelativeUri(); + std::filesystem::path RelPath{Uri.empty() ? "index.html" : Uri}; + std::filesystem::path AbsPath = m_Directory / RelPath; + + FileContents File = ReadFile(AbsPath); + + if (!File.ErrorCode) + { + // TODO: Map file extension to MIME type + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, File.Data[0]); + } + else + { + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Ooops!"sv); + } + } +} + +} // namespace zen diff --git a/zenserver/frontend/frontend.h b/zenserver/frontend/frontend.h new file mode 100644 index 000000000..2ae20e940 --- /dev/null +++ b/zenserver/frontend/frontend.h @@ -0,0 +1,24 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +namespace zen { + +class HttpFrontendService final : public zen::HttpService +{ +public: + HttpFrontendService(std::filesystem::path Directory); + virtual ~HttpFrontendService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + std::filesystem::path m_Directory; +}; + +} // namespace zen diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 576c88cb8..a684272c4 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -106,7 +106,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include "cache/structuredcachestore.h" #include "compute/apply.h" #include "diag/diagsvcs.h" -#include "experimental/frontend.h" +#include "frontend/frontend.h" #include "experimental/usnjournal.h" #include "monitoring/httpstats.h" #include "monitoring/httpstatus.h" -- cgit v1.2.3 From 7a13c8b127f3527f7e2a2c58840327029efb0e93 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Tue, 1 Mar 2022 13:14:01 +0100 Subject: Set a default value for "Size" parameter of IoBuffer(OuterBuffer, ...) --- zencore/include/zencore/iobuffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 5dbbb95bf..2802194a6 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -336,7 +336,7 @@ public: /** Create a buffer which references a sequence of bytes inside another buffer */ - ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes); + ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes=~0ull); /** Create a buffer which references a range of bytes which we assume will live * for the entire life time. -- cgit v1.2.3 From 8e419a9cd6d96f4ff25f202a4f5e4e0f9725a53c Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Tue, 1 Mar 2022 13:14:34 +0100 Subject: Marked IoBuffer::operator bool () method as explicit --- zencore/include/zencore/iobuffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 2802194a6..da68be516 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -351,7 +351,7 @@ public: ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); - inline operator bool() const { return !m_Core->IsNull(); } + inline explicit operator bool() const { return !m_Core->IsNull(); } inline operator MemoryView() const& { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); } inline void MakeOwned() { return m_Core->MakeOwned(); } [[nodiscard]] inline bool IsOwned() const { return m_Core->IsOwned(); } -- cgit v1.2.3 From 00956b29d36a57d398a93ef828af660b401afa21 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Tue, 1 Mar 2022 13:15:56 +0100 Subject: ZipFs class providing a rudimentary in-memory/memory-mapped file system --- zenserver/frontend/zipfs.cpp | 170 +++++++++++++++++++++++++++++++++++++++++++ zenserver/frontend/zipfs.h | 24 ++++++ 2 files changed, 194 insertions(+) create mode 100644 zenserver/frontend/zipfs.cpp create mode 100644 zenserver/frontend/zipfs.h diff --git a/zenserver/frontend/zipfs.cpp b/zenserver/frontend/zipfs.cpp new file mode 100644 index 000000000..5fb9d0177 --- /dev/null +++ b/zenserver/frontend/zipfs.cpp @@ -0,0 +1,170 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zipfs.h" + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +namespace { + +#if ZEN_COMPILER_MSC +# pragma warning(push) +# pragma warning(disable : 4200) +#endif + +using ZipInt16 = uint16_t; + +struct ZipInt32 +{ + operator uint32_t () const { return *(uint32_t*)Parts; } + uint16_t Parts[2]; +}; + +struct EocdRecord +{ + enum : uint32_t { + Magic = 0x0605'4b50, + }; + ZipInt32 Signature; + ZipInt16 ThisDiskIndex; + ZipInt16 CdStartDiskIndex; + ZipInt16 CdRecordThisDiskCount; + ZipInt16 CdRecordCount; + ZipInt32 CdSize; + ZipInt32 CdOffset; + ZipInt16 CommentSize; + char Comment[]; +}; + +struct CentralDirectoryRecord +{ + enum : uint32_t { + Magic = 0x0201'4b50, + }; + + ZipInt32 Signature; + ZipInt16 VersionMadeBy; + ZipInt16 VersionRequired; + ZipInt16 Flags; + ZipInt16 CompressionMethod; + ZipInt16 LastModTime; + ZipInt16 LastModDate; + ZipInt32 Crc32; + ZipInt32 CompressedSize; + ZipInt32 OriginalSize; + ZipInt16 FileNameLength; + ZipInt16 ExtraFieldLength; + ZipInt16 CommentLength; + ZipInt16 DiskIndex; + ZipInt16 InternalFileAttr; + ZipInt32 ExternalFileAttr; + ZipInt32 Offset; + char FileName[]; +}; + +struct LocalFileHeader +{ + enum : uint32_t { + Magic = 0x0304'4b50, + }; + + ZipInt32 Signature; + ZipInt16 VersionRequired; + ZipInt16 Flags; + ZipInt16 CompressionMethod; + ZipInt16 LastModTime; + ZipInt16 LastModDate; + ZipInt32 Crc32; + ZipInt32 CompressedSize; + ZipInt32 OriginalSize; + ZipInt16 FileNameLength; + ZipInt16 ExtraFieldLength; + char FileName[]; +}; + +#if ZEN_COMPILER_MSC +# pragma warning(pop) +#endif + +} // namespace + + + +////////////////////////////////////////////////////////////////////////// +ZipFs::ZipFs(IoBuffer&& Buffer) +{ + MemoryView View = Buffer.GetView(); + + uint8_t* Cursor = (uint8_t*)(View.GetData()) + View.GetSize(); + if (View.GetSize() < sizeof(EocdRecord)) + { + return; + } + + const auto* EocdCursor = (EocdRecord*)(Cursor - sizeof(EocdRecord)); + + // It is more correct to search backwards for EocdRecord::Magic as the + // comment can be of a variable length. But here we're not going to support + // zip files with comments. + if (EocdCursor->Signature != EocdRecord::Magic) + { + return; + } + + // Zip64 isn't supported either + if (EocdCursor->ThisDiskIndex == 0xffff) + { + return; + } + + Cursor -= View.GetSize(); + + const auto* CdCursor = (CentralDirectoryRecord*)(Cursor + EocdCursor->CdOffset); + for (int i = 0, n = EocdCursor->CdRecordCount; i < n; ++i) + { + const CentralDirectoryRecord& Cd = *CdCursor; + + bool Acceptable = true; + Acceptable &= (Cd.OriginalSize > 0); // has some content + Acceptable &= (Cd.CompressionMethod == 0); // is stored uncomrpessed + if (Acceptable) + { + const uint8_t* Lfh = Cursor + Cd.Offset; + if (uintptr_t(Lfh - Cursor) < View.GetSize()) + { + std::string_view FileName(Cd.FileName, Cd.FileNameLength); + m_Files.insert(std::make_pair(FileName, FileItem{Lfh, size_t(0)})); + } + } + + uint32_t ExtraBytes = Cd.FileNameLength + Cd.ExtraFieldLength + Cd.CommentLength; + CdCursor = (CentralDirectoryRecord*)(Cd.FileName + ExtraBytes); + } + + m_Buffer = std::move(Buffer); +} + +////////////////////////////////////////////////////////////////////////// +IoBuffer ZipFs::GetFile(const std::string_view& FileName) const +{ + FileMap::iterator Iter = m_Files.find(FileName); + if (Iter == m_Files.end()) + { + return{}; + } + + FileItem& Item = Iter->second; + if (Item.GetSize() > 0) + { + return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); + } + + const auto* Lfh = (LocalFileHeader*)(Item.GetData()); + Item = MemoryView( + Lfh->FileName + Lfh->FileNameLength + Lfh->ExtraFieldLength, + Lfh->OriginalSize + ); + return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); +} + +} // namespace zen diff --git a/zenserver/frontend/zipfs.h b/zenserver/frontend/zipfs.h new file mode 100644 index 000000000..a304e9ff5 --- /dev/null +++ b/zenserver/frontend/zipfs.h @@ -0,0 +1,24 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +class ZipFs +{ +public: + ZipFs() = default; + ZipFs(IoBuffer&& Buffer); + IoBuffer GetFile(const std::string_view& FileName) const; + +private: + using FileItem = MemoryView; + using FileMap = std::unordered_map; + FileMap mutable m_Files; + IoBuffer m_Buffer; +}; + +} // namespace zen -- cgit v1.2.3 From 57c32ad6d2534e975fb62d8c36768c72f06877f6 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Tue, 1 Mar 2022 14:34:23 +0100 Subject: Added some new mime types; javascript, css, png and ico --- zencore/include/zencore/iobuffer.h | 13 +++++++++++++ zenhttp/httpserver.cpp | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index da68be516..449236127 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -27,6 +27,10 @@ enum class ZenContentType : uint8_t kCompressedBinary = 7, kUnknownContentType = 8, kHTML = 9, + kJavaScript = 10, + kCSS = 11, + kPNG = 12, + kIcon = 13, kCOUNT }; @@ -58,6 +62,15 @@ ToString(ZenContentType ContentType) return "yaml"sv; case ZenContentType::kHTML: return "html"sv; + case ZenContentType::kJavaScript: + return "javascript"sv; + case ZenContentType::kCSS: + return "css"sv; + case ZenContentType::kPNG: + return "png"sv; + case ZenContentType::kIcon: + return "icon"sv; + } } diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp index c8e11468e..710b6f356 100644 --- a/zenhttp/httpserver.cpp +++ b/zenhttp/httpserver.cpp @@ -60,6 +60,18 @@ MapContentTypeToString(HttpContentType ContentType) case HttpContentType::kHTML: return "text/html"sv; + + case HttpContentType::kJavaScript: + return "application/javascript"sv; + + case HttpContentType::kCSS: + return "text/css"sv; + + case HttpContentType::kPNG: + return "image/png"sv; + + case HttpContentType::kIcon: + return "image/x-icon"sv; } } @@ -76,6 +88,10 @@ static constinit uint32_t HashCompressedBinary = HashStringDjb2("application/x static constinit uint32_t HashJson = HashStringDjb2("json"sv); static constinit uint32_t HashYaml = HashStringDjb2("yaml"sv); static constinit uint32_t HashHtml = HashStringDjb2("text/html"sv); +static constinit uint32_t HashJavaScript = HashStringDjb2("application/javascript"sv); +static constinit uint32_t HashCss = HashStringDjb2("text/css"sv); +static constinit uint32_t HashPng = HashStringDjb2("image/png"sv); +static constinit uint32_t HashIcon = HashStringDjb2("image/x-icon"sv); std::once_flag InitContentTypeLookup; @@ -96,6 +112,10 @@ struct HashedTypeEntry {HashText, HttpContentType::kText}, {HashCompressedBinary, HttpContentType::kCompressedBinary}, {HashHtml, HttpContentType::kHTML}, + {HashJavaScript, HttpContentType::kJavaScript}, + {HashCss, HttpContentType::kCSS}, + {HashPng, HttpContentType::kPNG}, + {HashIcon, HttpContentType::kIcon}, // clang-format on }; -- cgit v1.2.3 From 6a955b38cfcc3a566b332f1d1fcc630f7a2f641f Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 2 Mar 2022 09:19:25 +0100 Subject: Function to find .zip appended to a IoBuffer-wrapped executable file --- zenserver/frontend/frontend.cpp | 97 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp index 4bd3ec90a..0611f068c 100644 --- a/zenserver/frontend/frontend.cpp +++ b/zenserver/frontend/frontend.cpp @@ -5,8 +5,105 @@ #include #include +ZEN_THIRD_PARTY_INCLUDES_START +#if ZEN_PLATFORM_WINDOWS +# include +#endif +ZEN_THIRD_PARTY_INCLUDES_END + namespace zen { +////////////////////////////////////////////////////////////////////////// +static IoBuffer FindZipFsInBinary(const IoBuffer& BinBuffer) +{ + if (BinBuffer.GetSize() < 4) + { + return {}; + } + + uintptr_t Cursor = uintptr_t(BinBuffer.GetData()); + size_t BinSize = 0; + + uint32_t Magic = *(uint32_t*)(BinBuffer.GetData()); +#if ZEN_PLATFORM_LINUX + if (Magic == 0x464c457f) + { + struct Elf64Header + { + char Ident[16]; + uint16_t Type; + uint16_t Machine; + uint32_t Version; + uint64_t Entry; + uint64_t ProgHeaderOffset; + uint64_t SectionHeaderOffset; + uint32_t Flags; + uint16_t EhSize; + uint16_t ProgHeaderEntrySize; + uint16_t ProgHeaderCount; + uint16_t SectionHeaderEntrySize; + uint16_t SectionHeaderCount; + uint16_t SectionStringIndex; + }; + + struct SectionHeader + { + uint32_t NameIndex; + uint32_t Type; + uint64_t Flags; + uint64_t Address; + uint64_t Offset; + uint64_t Size; + uint64_t _Other[3]; + }; + + const auto* Elf = (Elf64Header*)Cursor; + if (Elf->Ident[4] != 0x02) // Elf64 + { + return {}; + } + + const auto* Section = (SectionHeader*)(Cursor + Elf->SectionHeaderOffset); + + /* + size_t BinSize = 0; + for (int i = 0, n = Elf->SectionHeaderCount; i < n; ++i, ++Section) + { + uint32_t SectionEnd = Section->Offset + Section->Size; + BinSize = (SectionEnd > BinSize) ? SectionEnd : BinSize; + } + */ + + // What if the section headers aren't the last thing in the fiile though? + BinSize = Elf->SectionHeaderEntrySize; + BinSize *= Elf->SectionHeaderCount; + BinSize += Elf->SectionHeaderOffset; + } +#elif ZEN_PLATFORM_WINDOWS + if ((Magic & 0xffff) == 0x5a4d) + { + const auto* Dos = (IMAGE_DOS_HEADER*)Cursor; + const auto* Nt = (IMAGE_NT_HEADERS64*)(Cursor + Dos->e_lfanew); + const auto* Section = (IMAGE_SECTION_HEADER*)(uintptr_t(&Nt->OptionalHeader) + Nt->FileHeader.SizeOfOptionalHeader); + + for (int i = 0, n = Nt->FileHeader.NumberOfSections; i < n; ++i, ++Section) + { + uint32_t SectionEnd = Section->PointerToRawData + Section->SizeOfRawData; + BinSize = (SectionEnd > BinSize) ? SectionEnd : BinSize; + } + } +#elif ZEN_PLATFORM_MAC +# error "needs to be implemented!" +#endif // win/linux/mac + + if (!BinSize || BinSize > BinBuffer.GetSize()) + { + return {}; + } + + return IoBuffer(BinBuffer, BinSize); +} + namespace html { constexpr std::string_view Index = R"( -- cgit v1.2.3 From 27b351c2bac99d06f020f60fd859ecc93d4fd140 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 2 Mar 2022 10:18:20 +0100 Subject: Implemented zip file locating for fat Mach-o binaries --- zenserver/frontend/frontend.cpp | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp index 0611f068c..17614f163 100644 --- a/zenserver/frontend/frontend.cpp +++ b/zenserver/frontend/frontend.cpp @@ -2,6 +2,7 @@ #include "frontend.h" +#include #include #include @@ -93,7 +94,38 @@ static IoBuffer FindZipFsInBinary(const IoBuffer& BinBuffer) } } #elif ZEN_PLATFORM_MAC -# error "needs to be implemented!" + if (Magic == 0xbebafeca) + { + struct MachInt32 + { + operator uint32_t () const { return ByteSwap(Value); } + uint32_t Value; + }; + + struct MachFatArch + { + MachInt32 CpuType; + MachInt32 SubType; + MachInt32 Offset; + MachInt32 Size; + MachInt32 Alignment; + }; + + struct MachFatHeader + { + uint32_t Magic; + MachInt32 NumArchs; + MachFatArch Archs[]; + }; + + const auto* Header = (MachFatHeader*)Cursor; + for (int i = 0, n = Header->NumArchs; i < n; ++i) + { + const MachFatArch* Arch = Header->Archs + i; + uint32_t ArchEnd = Arch->Offset + Arch->Size; + BinSize = (ArchEnd > BinSize) ? ArchEnd : BinSize; + } + } #endif // win/linux/mac if (!BinSize || BinSize > BinBuffer.GetSize()) -- cgit v1.2.3 From 9dd7b73739486d4de3206c79f497231b286f53e6 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 2 Mar 2022 10:30:00 +0100 Subject: Serve the /dashboard endpoint from an embedded .zip file --- zenserver/frontend/frontend.cpp | 162 +++++++++++++++++++------------------ zenserver/frontend/frontend.h | 13 +-- zenserver/frontend/html/index.html | 59 ++++++++++++++ 3 files changed, 148 insertions(+), 86 deletions(-) create mode 100644 zenserver/frontend/html/index.html diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp index 17614f163..b87d7e313 100644 --- a/zenserver/frontend/frontend.cpp +++ b/zenserver/frontend/frontend.cpp @@ -136,113 +136,115 @@ static IoBuffer FindZipFsInBinary(const IoBuffer& BinBuffer) return IoBuffer(BinBuffer, BinSize); } -namespace html { - - constexpr std::string_view Index = R"( - - - - - - - - - - -
-
-
-
-__________                  _________  __                           
-\____    /  ____    ____   /   _____/_/  |_   ____  _______   ____  
-  /     / _/ __ \  /    \  \_____  \ \   __\ /  _ \ \_  __ \_/ __ \
- /     /_ \  ___/ |   |  \ /        \ |  |  (  <_> ) |  | \/\  ___/ 
-/_______ \ \___  >|___|  //_______  / |__|   \____/  |__|    \___  >
-        \/     \/      \/         \/                             \/ 
-				
-
-			
-
-
-
Z$:
-

-		
-
- - -)"; - -} // namespace html - -HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory) +//////////////////////////////////////////////////////////////////////////////// +HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) +: m_Directory(Directory) { + std::filesystem::path SelfPath = GetRunningExecutablePath(); + + // Locate a .zip file appended onto the end of this binary + IoBuffer SelfBuffer = IoBufferBuilder::MakeFromFile(SelfPath); + IoBuffer SelfTailBuffer = FindZipFsInBinary(SelfBuffer); + if (SelfTailBuffer) + { + m_ZipFs = ZipFs(std::move(SelfTailBuffer)); + } + +#if ZEN_BUILD_DEBUG + if (!Directory.empty()) + { + return; + } + + std::error_code ErrorCode; + for (auto Path = SelfPath.parent_path(); !Path.empty(); Path = Path.parent_path()) + { + if (!std::filesystem::is_regular_file(Path / "xmake.lua", ErrorCode)) + { + continue; + } + + auto HtmlDir = (Path / __FILE__).parent_path() / "html"; + if (std::filesystem::is_directory(HtmlDir, ErrorCode)) + { + m_Directory = HtmlDir; + } + break; + } +#endif } +//////////////////////////////////////////////////////////////////////////////// HttpFrontendService::~HttpFrontendService() { } +//////////////////////////////////////////////////////////////////////////////// const char* HttpFrontendService::BaseUri() const { return "/dashboard"; // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp } +//////////////////////////////////////////////////////////////////////////////// void HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) { using namespace std::literals; - if (m_Directory.empty()) + std::string_view Uri = Request.RelativeUri(); + for (; Uri[0] == '/'; Uri = Uri.substr(1)); + if (Uri.empty()) + { + Uri = "index.html"sv; + } + + // Dismiss if the URI contains .. anywhere to prevent arbitrary file reads + if (Uri.find("..") != Uri.npos) { - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, html::Index); + Request.WriteResponse(HttpResponseCode::Forbidden); + return; } - else + + // Map the file extension to a MIME type. To keep things constrained, only a + // small subset of file extensions is allowed. + HttpContentType ContentType = HttpContentType::kCOUNT; + size_t DotIndex = Uri.rfind("."); + if (DotIndex != Uri.npos) { - std::string_view Uri = Request.RelativeUri(); - std::filesystem::path RelPath{Uri.empty() ? "index.html" : Uri}; - std::filesystem::path AbsPath = m_Directory / RelPath; + const std::string_view DotExt = Uri.substr(DotIndex); + if (DotExt == ".html") ContentType = HttpContentType::kHTML; + else if (DotExt == ".js") ContentType = HttpContentType::kJSON; + else if (DotExt == ".css") ContentType = HttpContentType::kCSS; + else if (DotExt == ".png") ContentType = HttpContentType::kPNG; + else if (DotExt == ".ico") ContentType = HttpContentType::kIcon; + } - FileContents File = ReadFile(AbsPath); + if (ContentType == HttpContentType::kCOUNT) + { + Request.WriteResponse(HttpResponseCode::Forbidden); + return; + } + // The given content directory overrides any zip-fs discovered in the binary + if (!m_Directory.empty()) + { + FileContents File = ReadFile(m_Directory / Uri); if (!File.ErrorCode) { - // TODO: Map file extension to MIME type - Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, File.Data[0]); - } - else - { - return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Ooops!"sv); + Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]); + return; } } + + IoBuffer FileBuffer = m_ZipFs.GetFile(Uri); + if (FileBuffer) + { + Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer); + return; + } + + Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); } } // namespace zen diff --git a/zenserver/frontend/frontend.h b/zenserver/frontend/frontend.h index 2ae20e940..bf5298169 100644 --- a/zenserver/frontend/frontend.h +++ b/zenserver/frontend/frontend.h @@ -3,6 +3,7 @@ #pragma once #include +#include "zipfs.h" #include @@ -11,14 +12,14 @@ namespace zen { class HttpFrontendService final : public zen::HttpService { public: - HttpFrontendService(std::filesystem::path Directory); - virtual ~HttpFrontendService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; + HttpFrontendService(std::filesystem::path Directory); + virtual ~HttpFrontendService(); + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; private: - std::filesystem::path m_Directory; + ZipFs m_ZipFs; + std::filesystem::path m_Directory; }; } // namespace zen diff --git a/zenserver/frontend/html/index.html b/zenserver/frontend/html/index.html new file mode 100644 index 000000000..252ee621e --- /dev/null +++ b/zenserver/frontend/html/index.html @@ -0,0 +1,59 @@ + + + + + + + + + + +
+
+
+
+__________                  _________  __
+\____    /  ____    ____   /   _____/_/  |_   ____  _______   ____
+  /     / _/ __ \  /    \  \_____  \ \   __\ /  _ \ \_  __ \_/ __ \
+ /     /_ \  ___/ |   |  \ /        \ |  |  (  <_> ) |  | \/\  ___/
+/_______ \ \___  >|___|  //_______  / |__|   \____/  |__|    \___  >
+        \/     \/      \/         \/                             \/
+				
+
+			
+
+
+
Z$:
+

+		
+
+ + -- cgit v1.2.3 From f8170856b601089ba89a69047ccbcd57f16b872b Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 3 Mar 2022 13:23:42 +0100 Subject: Allow nil arguments when bundle.lua launches processes --- scripts/bundle.lua | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/bundle.lua b/scripts/bundle.lua index 90fb13657..e2e43cd87 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -2,8 +2,15 @@ -------------------------------------------------------------------------------- local function _exec(cmd, ...) - print("--", cmd, ...) - local ret = os.execv(cmd, {...}) + local args = {} + for _, arg in pairs({...}) do + if arg then + table.insert(args, arg) + end + end + + print("--", cmd, table.unpack(args)) + local ret = os.execv(cmd, args) print() return ret end -- cgit v1.2.3 From 7c4d93104f51cc4ad4f9c643a308aa0551f40cd9 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 3 Mar 2022 13:24:45 +0100 Subject: Zip up the frontend content and append to executable when bundling --- scripts/bundle.lua | 93 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 12 deletions(-) diff --git a/scripts/bundle.lua b/scripts/bundle.lua index e2e43cd87..238a04bc1 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -42,23 +42,52 @@ local function _build(arch, debug, config_args) end -------------------------------------------------------------------------------- -local function _zip(zip_path, ...) +local function _zip(store_only, zip_path, ...) + -- Here's the rules; if len(...) is 1 and it is a dir then create a zip with + -- archive paths like this; + -- + -- glob(foo/bar/**) -> foo/bar/abc, foo/bar/dir/123 -> zip(abc, dir/123) + -- + -- Otherwise assume ... is file paths and add without leading directories; + -- + -- foo/abc, bar/123 -> zip(abc, 123) + + zip_path = path.absolute(zip_path) os.tryrm(zip_path) + local inputs = {...} + + local source_dir = nil + if #inputs == 1 and os.isdir(inputs[1]) then + source_dir = inputs[1] + end + import("detect.tools.find_7z") local cmd = find_7z() if cmd then - -- A ./ prefix makes 7z ignore the directory structure input_paths = {} - for _, input_path in ipairs({...}) do - input_path = path.relative(input_path, ".") - if input_path:sub(2,2) ~= ":" then - input_path = "./"..input_path + if source_dir then + -- Suffixing a directory path with a "/." will have 7z set the path + -- for archived files relative to that directory. + input_paths = { path.join(source_dir, ".") } + else + for _, input_path in pairs(inputs) do + -- If there is a "/./" anywhere in file paths then 7z drops all + -- directory information and just archives the file by name + input_path = path.relative(input_path, ".") + if input_path:sub(2,2) ~= ":" then + input_path = "./"..input_path + end + table.insert(input_paths, input_path) end - table.insert(input_paths, input_path) end - local ret = _exec("7z", "a", zip_path, table.unpack(input_paths)) + compression_level = "-mx9" + if store_only then + compression_level = "-mx0" + end + + local ret = _exec("7z", "a", "-r", compression_level, zip_path, table.unpack(input_paths)) if ret > 0 then raise("Received error from 7z") end @@ -69,10 +98,29 @@ local function _zip(zip_path, ...) import("detect.tools.find_zip") cmd = find_zip() if cmd then - local ret = _exec("zip", "--junk-paths", zip_path, ...) + local input_paths = inputs + local cwd = os.curdir() + if source_dir then + os.cd(source_dir) + input_paths = { "." } + end + + compression_level = "-9" + if store_only then + compression_level = "-0" + end + + local strip_leading_path = nil + if not source_dir then + strip_leading_path = "--junk-paths" + end + + local ret = _exec("zip", "-r", compression_level, strip_leading_path, zip_path, table.unpack(input_paths)) if ret > 0 then raise("Received error from zip") end + + os.cd(cwd) return end print("zip not found") @@ -96,6 +144,21 @@ local function _find_vcpkg_binary(triple, port, binary) return bin_path end +-------------------------------------------------------------------------------- +local function _append_content_zip(bin_path) + local zip_path = "build/frontend.zip" + local content_dir = "zenserver/frontend/html/" + _zip(true, zip_path, content_dir) + + zip_file = io.open(zip_path, "rb") + local zip_data = zip_file:read("*all") + zip_file:close() + + bin_file = io.open(bin_path, "ab") + bin_file:write(zip_data) + bin_file:close() +end + -------------------------------------------------------------------------------- local function main_windows() import("core.base.option") @@ -109,12 +172,14 @@ local function main_windows() _build("x64", false, config_args) + _append_content_zip("build/windows/x64/release/zenserver.exe") + local crashpad_handler_path = _find_vcpkg_binary( "x64-windows-static", "sentry-native", "crashpad_handler.exe") - _zip( + _zip(false, zip_path, "build/windows/x64/release/zenserver.exe", "build/windows/x64/release/zenserver.pdb", @@ -139,6 +204,8 @@ local function main_mac() raise("Failed creating univeral binary") end + _append_content_zip("build/macosx/universal/release/zenserver") + -- At the time of writing vcpkg does not support sentry-native on arm64. Once -- it does we can create a univeral binary for this. For now just bundle x64 local crashpad_handler_path = _find_vcpkg_binary( @@ -147,7 +214,7 @@ local function main_mac() "crashpad_handler") -- Zip - _zip( + _zip(false, "build/zenserver-macos.zip", "build/macosx/universal/release/zenserver", crashpad_handler_path) @@ -166,9 +233,11 @@ local function main_linux() "crashpad_handler") --]] + _append_content_zip("build/linux/x86_64/release/zenserver") + _exec("scripts/bundle_linux.sh") - _zip( + _zip(false, "build/zenserver-linux.zip", "build/appimage/zenserver", crashpad_handler_path) -- cgit v1.2.3 From c057edd241483139107365426873a6dcf825f1af Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 3 Mar 2022 13:25:09 +0100 Subject: Fixed typo --- scripts/bundle.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bundle.lua b/scripts/bundle.lua index 238a04bc1..01268ab14 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -201,7 +201,7 @@ local function main_mac() "build/macosx/arm64/release/zenserver" ) if ret > 0 then - raise("Failed creating univeral binary") + raise("Failed creating universal binary") end _append_content_zip("build/macosx/universal/release/zenserver") -- cgit v1.2.3 From 130b9d574b19768b77a29c6e1d6452eee6287fef Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 16 Mar 2022 14:47:20 +0100 Subject: Corrected linux install Use same version for both download and install of xmake --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f5dd4ddb..7243a5182 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ date `.deb` files; ``` wget https://github.com/xmake-io/xmake/releases/download/v2.6.2/xmake-v2.6.2.amd64.deb -sudo dpkg -i xmake-v2.5.7.amd64.deb +sudo dpkg -i xmake-v2.6.2.amd64.deb xmake --version ``` -- cgit v1.2.3 From e44e554f02748d79934b605b13cc2e521c1edfa6 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 16 Mar 2022 14:59:51 +0100 Subject: Bump xmake install instructions to 2.6.4 2.6.2 can't run the current build script, 2.6.4 is required. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7243a5182..5b85d86f9 100644 --- a/README.md +++ b/README.md @@ -96,8 +96,8 @@ Next we need the `xmake` build system. For this we will download and install date `.deb` files; ``` -wget https://github.com/xmake-io/xmake/releases/download/v2.6.2/xmake-v2.6.2.amd64.deb -sudo dpkg -i xmake-v2.6.2.amd64.deb +wget https://github.com/xmake-io/xmake/releases/download/v2.6.4/xmake-v2.6.4.amd64.deb +sudo dpkg -i xmake-v2.6.4.amd64.deb xmake --version ``` -- cgit v1.2.3 From f68e7e6fd8510e53360423daeb0c9e535caaeb5d Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 17 Mar 2022 10:31:19 +0100 Subject: FIxed up .clang-format setting for enums clang-format 12.0.0 which we used previously did not respect the AfterEnum setting, which lead to reformatting when using more recent builds. This change updates the setting to match the old behaviour --- .clang-format | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.clang-format b/.clang-format index f1b47c7d7..38336c39c 100644 --- a/.clang-format +++ b/.clang-format @@ -31,7 +31,7 @@ BraceWrapping: AfterCaseLabel: true AfterClass: true AfterControlStatement: Always - AfterEnum: false + AfterEnum: true AfterFunction: true AfterNamespace: false AfterObjCDeclaration: false -- cgit v1.2.3 From 4e2908f971cc137aa70c457bcbc46a54293378ed Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 17 Mar 2022 13:37:56 +0100 Subject: Some more updates to .clang-format to remove redundant settings --- .clang-format | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/.clang-format b/.clang-format index 38336c39c..4688f60ae 100644 --- a/.clang-format +++ b/.clang-format @@ -66,10 +66,6 @@ DerivePointerAlignment: false DisableFormat: false ExperimentalAutoDetectBinPacking: false FixNamespaceComments: true -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH IncludeBlocks: Preserve IncludeCategories: - Regex: '^' @@ -118,31 +114,7 @@ PointerAlignment: Left RawStringFormats: - Language: Cpp Delimiters: - - cc - - CC - cpp - - Cpp - - CPP - - 'c++' - - 'C++' - CanonicalDelimiter: '' - BasedOnStyle: google - - Language: TextProto - Delimiters: - - pb - - PB - - proto - - PROTO - EnclosingFunctions: - - EqualsProto - - EquivToProto - - PARSE_PARTIAL_TEXT_PROTO - - PARSE_TEST_PROTO - - PARSE_TEXT_PROTO - - ParseTextOrDie - - ParseTextProtoOrDie - - ParseTestProto - - ParsePartialTestProto CanonicalDelimiter: '' BasedOnStyle: google ReflowComments: true @@ -169,14 +141,9 @@ SpacesInSquareBrackets: false SpaceBeforeSquareBrackets: false BitFieldColonSpacing: Both Standard: Auto -StatementMacros: - - Q_UNUSED - - QT_REQUIRE_VERSION TabWidth: 4 UseCRLF: false UseTab: Always WhitespaceSensitiveMacros: - STRINGIZE - - PP_STRINGIZE - - BOOST_PP_STRINGIZE ... -- cgit v1.2.3 From d4cfea4764b983bf8e992b7219583c1f86ff0194 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 17 Mar 2022 14:15:37 +0100 Subject: Introduced basic validation of the clang-format version This is a very primitive implementation, which could probably be improved. We should also raise an error if clang-format is not found --- scripts/formatcode.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/formatcode.py b/scripts/formatcode.py index dc13ae117..49a8753da 100644 --- a/scripts/formatcode.py +++ b/scripts/formatcode.py @@ -1,8 +1,9 @@ import argparse -import os import fileinput +import os import pathlib import re +import subprocess match_expressions = [] valid_extensions = [] @@ -89,6 +90,20 @@ def parse_match_expressions(wildcards, matches): print(f'Could not parse input --match expression \'{regex}\': {str(ex)}') quit() +def validate_clang_format(): + vstring = subprocess.check_output("clang-format --version", shell=True).decode().rstrip() + + match = re.search(r'(\d+)\.(\d+)(\.(\d+))?$', vstring) + if not match: + raise ValueError("invalid version number '%s'" % vstring) + + (major, minor, patch) = match.group(1, 2, 4) + + if int(major) < 13: + if int(minor) == 0: + if int(patch) < 1: + raise ValueError(f'invalid clang-format version -- we require at least v12.0.1') + def _main(): global root_dir, use_batching @@ -106,6 +121,8 @@ def _main(): root_dir = pathlib.Path(__file__).parent.parent.resolve() use_batching = options.use_batching + validate_clang_format() + while True: if (os.path.isfile(".clang-format")): scan_zen(".") -- cgit v1.2.3 From 7466cb93fbb9f4082dc253a328222dac8bbe58e4 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Thu, 17 Mar 2022 09:55:09 -0700 Subject: Update horde compute to use Jupiter for storage (#60) --- zenserver/compute/apply.cpp | 38 +++-- zenserver/compute/apply.h | 14 +- zenserver/config.cpp | 95 +++++++++++ zenserver/config.h | 15 ++ zenserver/upstream/jupiter.cpp | 16 +- zenserver/upstream/jupiter.h | 5 +- zenserver/upstream/upstreamapply.cpp | 295 ++++++++++++++++++++++------------- zenserver/upstream/upstreamapply.h | 21 ++- zenserver/zenserver.cpp | 81 ++++++++-- 9 files changed, 430 insertions(+), 150 deletions(-) diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index e4d5697fa..044078aa4 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -6,6 +6,7 @@ # include # include +# include # include # include # include @@ -173,10 +174,10 @@ SandboxedFunctionJob::GrantNamedObjectAccess(PWSTR ObjectName, SE_OBJECT_TYPE Ob .grfAccessMode = GRANT_ACCESS, .grfInheritance = grfInhericance, .Trustee = {.pMultipleTrustee = nullptr, - .MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE, - .TrusteeForm = TRUSTEE_IS_SID, - .TrusteeType = TRUSTEE_IS_GROUP, - .ptstrName = (PWSTR)m_AppContainerSid}}; + .MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE, + .TrusteeForm = TRUSTEE_IS_SID, + .TrusteeType = TRUSTEE_IS_GROUP, + .ptstrName = (PWSTR)m_AppContainerSid}}; PACL OldAcl = nullptr; @@ -328,24 +329,29 @@ SandboxedFunctionJob::SpawnJob(std::filesystem::path ExePath) //////////////////////////////////////////////////////////////////////////////// -HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& BaseDir) +HttpFunctionService::HttpFunctionService(CasStore& Store, + CidStore& InCidStore, + const std::filesystem::path& BaseDir, + const CloudCacheClientOptions& ComputeOptions, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const UpstreamAuthConfig& StorageAuthConfig, + AuthMgr& Mgr) : m_Log(logging::Get("apply")) , m_CasStore(Store) , m_CidStore(InCidStore) , m_SandboxPath(BaseDir / "scratch") , m_FunctionPath(BaseDir / "func") { - m_UpstreamApply = MakeUpstreamApply({}, m_CasStore, m_CidStore); - - CloudCacheAccessToken AccessToken{.Value = "ServiceAccount 0f8056b30bd0df0959be55fc3338159b6f938456d3474aed0087fb965268d079", - .ExpireTime = CloudCacheAccessToken::TimePoint::max()}; - - CloudCacheClientOptions Options = {.ServiceUrl = "https://horde.devtools-dev.epicgames.com"sv, - .DdcNamespace = "default"sv, - .BlobStoreNamespace = "default"sv}; - - auto HordeUpstreamEndpoint = - MakeHordeUpstreamEndpoint(Options, CloudCacheTokenProvider::CreateFromStaticToken(AccessToken), m_CasStore, m_CidStore); + m_UpstreamApply = UpstreamApply::Create({}, m_CasStore, m_CidStore); + + auto HordeUpstreamEndpoint = UpstreamApplyEndpoint::CreateHordeEndpoint(ComputeOptions, + ComputeAuthConfig, + StorageOptions, + StorageAuthConfig, + m_CasStore, + m_CidStore, + Mgr); m_UpstreamApply->RegisterEndpoint(std::move(HordeUpstreamEndpoint)); m_UpstreamApply->Initialize(); diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h index 161e47e06..e00afcd61 100644 --- a/zenserver/compute/apply.h +++ b/zenserver/compute/apply.h @@ -23,6 +23,11 @@ namespace zen { class CasStore; class CidStore; class UpstreamApply; +class CloudCacheClient; +class AuthMgr; + +struct UpstreamAuthConfig; +struct CloudCacheClientOptions; /** * Lambda style compute function service @@ -30,7 +35,14 @@ class UpstreamApply; class HttpFunctionService : public HttpService { public: - HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& SandboxBaseDir); + HttpFunctionService(CasStore& Store, + CidStore& InCidStore, + const std::filesystem::path& BaseDir, + const CloudCacheClientOptions& ComputeOptions, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const UpstreamAuthConfig& StorageAuthConfig, + AuthMgr& Mgr); ~HttpFunctionService(); virtual const char* BaseUri() const override; diff --git a/zenserver/config.cpp b/zenserver/config.cpp index bcacc16c0..adb079d83 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -345,6 +345,62 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"), ""); + options.add_option("compute", + "", + "upstream-horde-url", + "URL to a Horde instance.", + cxxopts::value(ServerOptions.UpstreamCacheConfig.HordeConfig.Url)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-oauth-url", + "URL to the OAuth provier", + cxxopts::value(ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthUrl)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-oauth-clientid", + "The OAuth client ID", + cxxopts::value(ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientId)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-oauth-clientsecret", + "The OAuth client secret", + cxxopts::value(ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientSecret)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-openid-provider", + "Name of a registered Open ID provider", + cxxopts::value(ServerOptions.UpstreamCacheConfig.HordeConfig.OpenIdProvider)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-token", + "A static authentication token", + cxxopts::value(ServerOptions.UpstreamCacheConfig.HordeConfig.AccessToken)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-cluster", + "The Horde compute cluster id", + cxxopts::value(ServerOptions.UpstreamCacheConfig.HordeConfig.Cluster)->default_value(""), + ""); + + options.add_option("compute", + "", + "upstream-horde-namespace", + "The Jupiter namespace to use with Horde compute", + cxxopts::value(ServerOptions.UpstreamCacheConfig.HordeConfig.Namespace)->default_value(""), + ""); + options.add_option("gc", "", "gc-enabled", @@ -596,6 +652,45 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio } } + if (sol::optional ComputeConfig = lua["compute"]) + { + ServerOptions.ComputeServiceEnabled = ComputeConfig->get_or("enable", ServerOptions.ComputeServiceEnabled); + + if (auto UpstreamConfig = ComputeConfig->get>("upstream")) + { + if (auto HordeConfig = UpstreamConfig->get>("horde")) + { + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("name"), + ServerOptions.UpstreamCacheConfig.HordeConfig.Name); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("url"), + ServerOptions.UpstreamCacheConfig.HordeConfig.Url); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("oauthprovider"), + ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthUrl); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("oauthclientid"), + ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientId); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("oauthclientsecret"), + ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientSecret); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("openidprovider"), + ServerOptions.UpstreamCacheConfig.HordeConfig.OpenIdProvider); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("token"), + ServerOptions.UpstreamCacheConfig.HordeConfig.AccessToken); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("cluster"), + ServerOptions.UpstreamCacheConfig.HordeConfig.Cluster); + UpdateStringValueFromConfig(HordeConfig.value(), + std::string_view("namespace"), + ServerOptions.UpstreamCacheConfig.HordeConfig.Namespace); + }; + } + } + if (sol::optional GcConfig = lua["gc"]) { ServerOptions.GcConfig.IntervalSeconds = GcConfig.value().get_or("intervalseconds", 0); diff --git a/zenserver/config.h b/zenserver/config.h index fd569bdb1..a7a7815a8 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -33,6 +33,19 @@ struct ZenUpstreamJupiterConfig bool UseLegacyDdc = false; }; +struct ZenUpstreamHordeConfig +{ + std::string Name; + std::string Url; + std::string OAuthUrl; + std::string OAuthClientId; + std::string OAuthClientSecret; + std::string OpenIdProvider; + std::string AccessToken; + std::string Cluster; + std::string Namespace; +}; + struct ZenUpstreamZenConfig { std::string Name; @@ -51,6 +64,7 @@ enum class UpstreamCachePolicy : uint8_t struct ZenUpstreamCacheConfig { ZenUpstreamJupiterConfig JupiterConfig; + ZenUpstreamHordeConfig HordeConfig; ZenUpstreamZenConfig ZenConfig; int32_t UpstreamThreadCount = 4; int32_t ConnectTimeoutMilliseconds = 5000; @@ -106,6 +120,7 @@ struct ZenServerOptions bool IsTest = false; bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements bool StructuredCacheEnabled = true; + bool ComputeServiceEnabled = true; bool ShouldCrash = false; // Option for testing crash handling bool IsFirstRun = false; bool NoSentry = false; diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 2b064a610..eef1daab0 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -577,12 +577,12 @@ CloudCacheSession::ObjectExists(const std::set& Keys) } CloudCacheResult -CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksData) +CloudCacheSession::PostComputeTasks(IoBuffer TasksData) { ZEN_TRACE_CPU("HordeClient::PostComputeTasks"); ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << m_CacheClient->ComputeCluster(); cpr::Session& Session = GetSession(); const CloudCacheAccessToken& AccessToken = GetAccessToken(); @@ -609,8 +609,11 @@ CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksDa CloudCacheResult CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t WaitSeconds) { + ZEN_TRACE_CPU("HordeClient::GetComputeUpdates"); + ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId << "/updates?wait=" << WaitSeconds; + Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << m_CacheClient->ComputeCluster() << "/updates/" << ChannelId + << "?wait=" << WaitSeconds; cpr::Session& Session = GetSession(); const CloudCacheAccessToken& AccessToken = GetAccessToken(); @@ -703,6 +706,8 @@ CloudCacheSession::VerifyAccessToken(long StatusCode) CloudCacheResult CloudCacheSession::CacheTypeExists(std::string_view TypeId, const IoHash& Key) { + ZEN_TRACE_CPU("HordeClient::CacheTypeExists"); + ExtendableStringBuilder<256> Uri; Uri << m_CacheClient->ServiceUrl() << "/api/v1/" << TypeId << "/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString(); @@ -731,6 +736,8 @@ CloudCacheSession::CacheTypeExists(std::string_view TypeId, const IoHash& Key) CloudCacheExistsResult CloudCacheSession::CacheTypeExists(std::string_view TypeId, const std::set& Keys) { + ZEN_TRACE_CPU("HordeClient::CacheTypeExists"); + ExtendableStringBuilder<256> Query; for (const auto& Key : Keys) { @@ -766,7 +773,7 @@ CloudCacheSession::CacheTypeExists(std::string_view TypeId, const std::set& Keys); CloudCacheExistsResult ObjectExists(const std::set& Keys); - CloudCacheResult PostComputeTasks(std::string_view ChannelId, IoBuffer TasksData); + CloudCacheResult PostComputeTasks(IoBuffer TasksData); CloudCacheResult GetComputeUpdates(std::string_view ChannelId, const uint32_t WaitSeconds = 0); CloudCacheResult GetObjectTree(const IoHash& Key); @@ -167,6 +167,7 @@ struct CloudCacheClientOptions std::string_view ServiceUrl; std::string_view DdcNamespace; std::string_view BlobStoreNamespace; + std::string_view ComputeCluster; std::chrono::milliseconds ConnectTimeout{5000}; std::chrono::milliseconds Timeout{}; bool UseLegacyDdc = false; @@ -184,6 +185,7 @@ public: CloudCacheAccessToken AcquireAccessToken(); std::string_view DdcNamespace() const { return m_DdcNamespace; } std::string_view BlobStoreNamespace() const { return m_BlobStoreNamespace; } + std::string_view ComputeCluster() const { return m_ComputeCluster; } std::string_view ServiceUrl() const { return m_ServiceUrl; } spdlog::logger& Logger() { return m_Log; } @@ -193,6 +195,7 @@ private: std::string m_ServiceUrl; std::string m_DdcNamespace; std::string m_BlobStoreNamespace; + std::string m_ComputeCluster; std::chrono::milliseconds m_ConnectTimeout{}; std::chrono::milliseconds m_Timeout{}; std::unique_ptr m_TokenProvider; diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index 63c334265..918697224 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -23,6 +23,9 @@ # include # include +# include +# include + # include "cache/structuredcachestore.h" # include "diag/logging.h" @@ -48,17 +51,76 @@ namespace detail { class HordeUpstreamApplyEndpoint final : public UpstreamApplyEndpoint { public: - HordeUpstreamApplyEndpoint(const CloudCacheClientOptions& Options, - std::unique_ptr TokenProvider, - CasStore& CasStore, - CidStore& CidStore) + HordeUpstreamApplyEndpoint(const CloudCacheClientOptions& ComputeOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& StorageAuthConfig, + CasStore& CasStore, + CidStore& CidStore, + AuthMgr& Mgr) : m_Log(logging::Get("upstream-apply")) + , m_AuthMgr(Mgr) , m_CasStore(CasStore) , m_CidStore(CidStore) { - m_DisplayName = fmt::format("Horde - '{}'", Options.ServiceUrl); - m_Client = new CloudCacheClient(Options, std::move(TokenProvider)); + m_DisplayName = fmt::format("{} - '{}'+'{}'", ComputeOptions.Name, ComputeOptions.ServiceUrl, StorageOptions.ServiceUrl); m_ChannelId = fmt::format("zen-{}", zen::GetSessionIdString()); + + { + std::unique_ptr TokenProvider; + + if (ComputeAuthConfig.OAuthUrl.empty() == false) + { + TokenProvider = + CloudCacheTokenProvider::CreateFromOAuthClientCredentials({.Url = ComputeAuthConfig.OAuthUrl, + .ClientId = ComputeAuthConfig.OAuthClientId, + .ClientSecret = ComputeAuthConfig.OAuthClientSecret}); + } + else if (ComputeAuthConfig.OpenIdProvider.empty() == false) + { + TokenProvider = + CloudCacheTokenProvider::CreateFromCallback([this, ProviderName = std::string(ComputeAuthConfig.OpenIdProvider)]() { + AuthMgr::OpenIdAccessToken Token = m_AuthMgr.GetOpenIdAccessToken(ProviderName); + return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; + }); + } + else + { + CloudCacheAccessToken AccessToken{.Value = std::string(ComputeAuthConfig.AccessToken), + .ExpireTime = CloudCacheAccessToken::TimePoint::max()}; + TokenProvider = CloudCacheTokenProvider::CreateFromStaticToken(AccessToken); + } + + m_Client = new CloudCacheClient(ComputeOptions, std::move(TokenProvider)); + } + + { + std::unique_ptr TokenProvider; + + if (StorageAuthConfig.OAuthUrl.empty() == false) + { + TokenProvider = + CloudCacheTokenProvider::CreateFromOAuthClientCredentials({.Url = StorageAuthConfig.OAuthUrl, + .ClientId = StorageAuthConfig.OAuthClientId, + .ClientSecret = StorageAuthConfig.OAuthClientSecret}); + } + else if (StorageAuthConfig.OpenIdProvider.empty() == false) + { + TokenProvider = + CloudCacheTokenProvider::CreateFromCallback([this, ProviderName = std::string(StorageAuthConfig.OpenIdProvider)]() { + AuthMgr::OpenIdAccessToken Token = m_AuthMgr.GetOpenIdAccessToken(ProviderName); + return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; + }); + } + else + { + CloudCacheAccessToken AccessToken{.Value = std::string(StorageAuthConfig.AccessToken), + .ExpireTime = CloudCacheAccessToken::TimePoint::max()}; + TokenProvider = CloudCacheTokenProvider::CreateFromStaticToken(AccessToken); + } + + m_StorageClient = new CloudCacheClient(StorageOptions, std::move(TokenProvider)); + } } virtual ~HordeUpstreamApplyEndpoint() = default; @@ -109,10 +171,11 @@ namespace detail { m_PendingTasks[UpstreamData.TaskId] = ApplyRecord; } - CloudCacheSession Session(m_Client); + CloudCacheSession ComputeSession(m_Client); + CloudCacheSession StorageSession(m_StorageClient); { - CloudCacheResult Result = BatchPutBlobsIfMissing(Session, UpstreamData.Blobs); + CloudCacheResult Result = BatchPutBlobsIfMissing(StorageSession, UpstreamData.Blobs); Bytes += Result.Bytes; ElapsedSeconds += Result.ElapsedSeconds; if (!Result.Success) @@ -126,7 +189,7 @@ namespace detail { } { - CloudCacheResult Result = BatchPutObjectsIfMissing(Session, UpstreamData.Objects); + CloudCacheResult Result = BatchPutObjectsIfMissing(StorageSession, UpstreamData.Objects); Bytes += Result.Bytes; ElapsedSeconds += Result.ElapsedSeconds; if (!Result.Success) @@ -136,17 +199,33 @@ namespace detail { .Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds}; } + + Result = StorageSession.PutRef("requests"sv, + UpstreamData.TaskId, + UpstreamData.Objects[UpstreamData.TaskId].GetBuffer().AsIoBuffer(), + ZenContentType::kCbObject); + + Bytes += Result.Bytes; + ElapsedSeconds += Result.ElapsedSeconds; + if (!Result.Success) + { + return {.Error{.ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1, + .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to add task ref"}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } UpstreamData.Objects.clear(); } CbObjectWriter Writer; + Writer.AddString("c"sv, m_ChannelId); Writer.AddObjectAttachment("r"sv, UpstreamData.RequirementsId); Writer.BeginArray("t"sv); Writer.AddObjectAttachment(UpstreamData.TaskId); Writer.EndArray(); IoBuffer TasksData = Writer.Save().GetBuffer().AsIoBuffer(); - CloudCacheResult Result = Session.PostComputeTasks(m_ChannelId, TasksData); + CloudCacheResult Result = ComputeSession.PostComputeTasks(TasksData); Bytes += Result.Bytes; ElapsedSeconds += Result.ElapsedSeconds; if (!Result.Success) @@ -214,8 +293,8 @@ namespace detail { { return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, - .ErrorCode = ExistsResult.ErrorCode ? ExistsResult.ErrorCode : -1, - .Reason = !ExistsResult.Reason.empty() ? std::move(ExistsResult.Reason) : "Failed to put blobs"}; + .ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1, + .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to put blobs"}; } } @@ -265,8 +344,8 @@ namespace detail { { return {.Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds, - .ErrorCode = ExistsResult.ErrorCode ? ExistsResult.ErrorCode : -1, - .Reason = !ExistsResult.Reason.empty() ? std::move(ExistsResult.Reason) : "Failed to put objects"}; + .ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1, + .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to put objects"}; } } @@ -330,9 +409,10 @@ namespace detail { try { - CloudCacheSession Session(m_Client); + CloudCacheSession ComputeSession(m_Client); + CloudCacheSession StorageSession(m_StorageClient); - CloudCacheResult UpdatesResult = Session.GetComputeUpdates(m_ChannelId); + CloudCacheResult UpdatesResult = ComputeSession.GetComputeUpdates(m_ChannelId); Bytes += UpdatesResult.Bytes; ElapsedSeconds += UpdatesResult.ElapsedSeconds; if (!UpdatesResult.Success) @@ -349,9 +429,6 @@ namespace detail { CbObject TaskStatus = LoadCompactBinaryObject(UpdatesResult.Response); - // zen::StringBuilder<4096> ObjStr; - // zen::CompactBinaryToJson(TaskStatus, ObjStr); - for (auto& It : TaskStatus["u"sv]) { CbObjectView Status = It.AsObjectView(); @@ -366,7 +443,7 @@ namespace detail { continue; } - const IoHash TaskId = Status["h"sv].AsObjectAttachment(); + const IoHash TaskId = Status["h"sv].AsHash(); IoHash WorkerId; IoHash ActionId; @@ -383,7 +460,7 @@ namespace detail { m_PendingTasks.erase(TaskIt); } - GetUpstreamApplyResult Result = ProcessTaskStatus(Status, Session); + GetUpstreamApplyResult Result = ProcessTaskStatus(Status, StorageSession); Bytes += Result.Bytes; ElapsedSeconds += Result.ElapsedSeconds; @@ -411,9 +488,11 @@ namespace detail { CasStore& m_CasStore; CidStore& m_CidStore; + AuthMgr& m_AuthMgr; spdlog::logger& m_Log; std::string m_DisplayName; RefPtr m_Client; + RefPtr m_StorageClient; UpstreamApplyEndpointStats m_Stats; std::atomic_bool m_HealthOk{false}; std::string m_ChannelId; @@ -453,9 +532,9 @@ namespace detail { return {.Error{.ErrorCode = -1, .Reason = fmt::format("Task {}", ComputeTaskOutcomeToString(Outcome))}}; } - const IoHash TaskId = TaskStatus["h"sv].AsObjectAttachment(); + const IoHash TaskId = TaskStatus["h"sv].AsHash(); const DateTime Time = TaskStatus["t"sv].AsDateTime(); - const IoHash ResultHash = TaskStatus["r"sv].AsObjectAttachment(); + const IoHash ResultHash = TaskStatus["r"sv].AsHash(); const std::string_view AgentId = TaskStatus["a"sv].AsString(); const std::string_view LeaseId = TaskStatus["l"sv].AsString(); @@ -463,105 +542,96 @@ namespace detail { double ElapsedSeconds{}; // Get Result object and all Object Attachments + Binary Attachment IDs - CloudCacheResult ObjectTreeResult = Session.GetObjectTree(ResultHash); - Bytes += ObjectTreeResult.Bytes; - ElapsedSeconds += ObjectTreeResult.ElapsedSeconds; + CloudCacheResult ObjectRefResult = Session.GetRef("responses"sv, ResultHash, ZenContentType::kCbObject); + Bytes += ObjectRefResult.Bytes; + ElapsedSeconds += ObjectRefResult.ElapsedSeconds; - if (!ObjectTreeResult.Success) + if (!ObjectRefResult.Success) { return {.Error{.ErrorCode = -1, .Reason = "Failed to get result object data"}, .Bytes = Bytes, .ElapsedSeconds = ElapsedSeconds}; } - std::map TreeObjectData; - std::map TreeBinaryData; + std::vector ObjectsToIterate; + std::map ObjectData; + std::map BinaryData; - MemoryView ResponseView = ObjectTreeResult.Response; - while (ResponseView.GetSize() > 0) - { - CbFieldView Field = CbFieldView(ResponseView.GetData()); - ResponseView += Field.GetSize(); + ObjectData[ResultHash] = ObjectRefResult.Response; + CbObject Object = LoadCompactBinaryObject(ObjectData[ResultHash]); + Object.IterateAttachments([&](CbFieldView Field) { if (Field.IsObjectAttachment()) { - const IoHash Hash = Field.AsObjectAttachment(); - Field = CbFieldView(ResponseView.GetData()); - ResponseView += Field.GetSize(); - if (!Field.IsObject()) // No data + const IoHash AttachmentHash = Field.AsObjectAttachment(); + if (!ObjectData.contains(AttachmentHash)) { - TreeObjectData[Hash] = {}; - continue; + ObjectsToIterate.push_back(AttachmentHash); } - MemoryView FieldView = Field.AsObjectView().GetView(); - - TreeObjectData[Hash] = IoBuffer(IoBuffer::Wrap, FieldView.GetData(), FieldView.GetSize()); } else if (Field.IsBinaryAttachment()) { - const IoHash Hash = Field.AsBinaryAttachment(); - TreeBinaryData[Hash] = {}; - } - else // Unknown type - { + const IoHash AttachmentHash = Field.AsBinaryAttachment(); + BinaryData[AttachmentHash] = {}; } - } + }); - for (auto& It : TreeObjectData) + while (!ObjectsToIterate.empty()) { - if (It.second.GetSize() == 0) + const IoHash Hash = ObjectsToIterate.back(); + ObjectsToIterate.pop_back(); + + CloudCacheResult ObjectResult = Session.GetObject(Hash); + Bytes += ObjectRefResult.Bytes; + ElapsedSeconds += ObjectRefResult.ElapsedSeconds; + if (!ObjectResult.Success) { - CloudCacheResult ObjectResult = Session.GetObject(It.first); - Bytes += ObjectTreeResult.Bytes; - ElapsedSeconds += ObjectTreeResult.ElapsedSeconds; - if (!ObjectTreeResult.Success) + return {.Error{.ErrorCode = ObjectResult.ErrorCode, .Reason = std::move(ObjectResult.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; + } + ObjectData[Hash] = std::move(ObjectResult.Response); + + CbObject IterateObject = LoadCompactBinaryObject(ObjectData[Hash]); + IterateObject.IterateAttachments([&](CbFieldView Field) { + if (Field.IsObjectAttachment()) { - return {.Error{.ErrorCode = ObjectResult.ErrorCode, .Reason = std::move(ObjectResult.Reason)}, - .Bytes = Bytes, - .ElapsedSeconds = ElapsedSeconds}; + const IoHash AttachmentHash = Field.AsObjectAttachment(); + if (!ObjectData.contains(AttachmentHash)) + { + ObjectsToIterate.push_back(AttachmentHash); + } } - - if (!ObjectResult.Success) + else if (Field.IsBinaryAttachment()) { - return {.Error{.ErrorCode = -1, .Reason = "Failed to get result object attachment data"}, - .Bytes = Bytes, - .ElapsedSeconds = ElapsedSeconds}; + const IoHash AttachmentHash = Field.AsBinaryAttachment(); + BinaryData[AttachmentHash] = {}; } - It.second = std::move(ObjectResult.Response); - } + }); } - for (auto& It : TreeBinaryData) + // Batch load all binary data + for (auto& It : BinaryData) { - if (It.second.GetSize() == 0) + CloudCacheResult BlobResult = Session.GetBlob(It.first); + Bytes += ObjectRefResult.Bytes; + ElapsedSeconds += ObjectRefResult.ElapsedSeconds; + if (!BlobResult.Success) { - CloudCacheResult BlobResult = Session.GetBlob(It.first); - Bytes += ObjectTreeResult.Bytes; - ElapsedSeconds += ObjectTreeResult.ElapsedSeconds; - if (!BlobResult.Success) - { - return {.Error{.ErrorCode = BlobResult.ErrorCode, .Reason = std::move(BlobResult.Reason)}, - .Bytes = Bytes, - .ElapsedSeconds = ElapsedSeconds}; - } - - if (!BlobResult.Success) - { - return {.Error{.ErrorCode = -1, .Reason = "Failed to get result binary attachment data"}, - .Bytes = Bytes, - .ElapsedSeconds = ElapsedSeconds}; - } - It.second = std::move(BlobResult.Response); + return {.Error{.ErrorCode = BlobResult.ErrorCode, .Reason = std::move(BlobResult.Reason)}, + .Bytes = Bytes, + .ElapsedSeconds = ElapsedSeconds}; } + It.second = std::move(BlobResult.Response); } - CbObject ResultObject = LoadCompactBinaryObject(TreeObjectData[ResultHash]); + CbObject ResultObject = LoadCompactBinaryObject(ObjectData[ResultHash]); int32_t ExitCode = ResultObject["e"sv].AsInt32(); IoHash StdOutHash = ResultObject["so"sv].AsBinaryAttachment(); IoHash StdErrHash = ResultObject["se"sv].AsBinaryAttachment(); IoHash OutputHash = ResultObject["o"sv].AsObjectAttachment(); - std::string StdOut = std::string((const char*)TreeBinaryData[StdOutHash].GetData(), TreeBinaryData[StdOutHash].GetSize()); - std::string StdErr = std::string((const char*)TreeBinaryData[StdErrHash].GetData(), TreeBinaryData[StdErrHash].GetSize()); + std::string StdOut = std::string((const char*)BinaryData[StdOutHash].GetData(), BinaryData[StdOutHash].GetSize()); + std::string StdErr = std::string((const char*)BinaryData[StdErrHash].GetData(), BinaryData[StdErrHash].GetSize()); if (ExitCode != 0) { @@ -572,7 +642,7 @@ namespace detail { .StdErr = std::move(StdErr)}; } - CbObject OutputObject = LoadCompactBinaryObject(TreeObjectData[OutputHash]); + CbObject OutputObject = LoadCompactBinaryObject(ObjectData[OutputHash]); // Get build.output IoHash BuildOutputId; @@ -583,7 +653,7 @@ namespace detail { if (FileObject["n"sv].AsString() == "Build.output"sv) { BuildOutputId = FileObject["h"sv].AsBinaryAttachment(); - BuildOutput = TreeBinaryData[BuildOutputId]; + BuildOutput = BinaryData[BuildOutputId]; break; } } @@ -604,7 +674,7 @@ namespace detail { const CbObjectView DirectoryObject = It.AsObjectView(); if (DirectoryObject["n"sv].AsString() == "Outputs"sv) { - OutputDirectoryTree = TreeObjectData[DirectoryObject["h"sv].AsObjectAttachment()]; + OutputDirectoryTree = ObjectData[DirectoryObject["h"sv].AsObjectAttachment()]; break; } } @@ -636,7 +706,7 @@ namespace detail { // Hash is the compressed data hash, and how it is stored in Horde IoHash CompressedId = FileObject["h"sv].AsBinaryAttachment(); - if (!TreeBinaryData.contains(CompressedId)) + if (!BinaryData.contains(CompressedId)) { Log().warn("Object attachment chunk not retrieved from Horde {}", CompressedId.ToHexString()); return {.Error{.ErrorCode = -1, .Reason = "Object attachment chunk not retrieved from Horde"}, @@ -659,7 +729,7 @@ namespace detail { } const IoHash& CompressedId = CidToCompressedId.at(DecompressedId); - if (!TreeBinaryData.contains(CompressedId)) + if (!BinaryData.contains(CompressedId)) { Log().warn("Missing output {} compressed {} uncompressed", CompressedId.ToHexString(), @@ -668,7 +738,7 @@ namespace detail { return; } - CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(TreeBinaryData[CompressedId])); + CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(BinaryData[CompressedId])); if (!AttachmentBuffer) { @@ -826,11 +896,11 @@ namespace detail { int64_t Memory = ApplyRecord.WorkerDescriptor["memory"sv].AsInt64(); bool Exclusive = ApplyRecord.WorkerDescriptor["exclusive"sv].AsBool(); - // TODO: Remove override when Horde accepts the UE style Host Platforms (Win64, Linux, Mac) std::string Condition = fmt::format("Platform == '{}'", HostPlatform); if (HostPlatform == "Win64") { - Condition += " && Pool == 'Win-RemoteExec'"; + // TODO + // Condition += " && Pool == 'Win-RemoteExec'"; } std::map Resources; @@ -1176,10 +1246,10 @@ struct UpstreamApplyStats ////////////////////////////////////////////////////////////////////////// -class DefaultUpstreamApply final : public UpstreamApply +class UpstreamApplyImpl final : public UpstreamApply { public: - DefaultUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) + UpstreamApplyImpl(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) : m_Log(logging::Get("upstream-apply")) , m_Options(Options) , m_CasStore(CasStore) @@ -1188,7 +1258,7 @@ public: { } - virtual ~DefaultUpstreamApply() { Shutdown(); } + virtual ~UpstreamApplyImpl() { Shutdown(); } virtual bool Initialize() override { @@ -1213,12 +1283,12 @@ public: for (uint32_t Idx = 0; Idx < m_Options.ThreadCount; Idx++) { - m_UpstreamThreads.emplace_back(&DefaultUpstreamApply::ProcessUpstreamQueue, this); + m_UpstreamThreads.emplace_back(&UpstreamApplyImpl::ProcessUpstreamQueue, this); } - m_UpstreamUpdatesThread = std::thread(&DefaultUpstreamApply::ProcessUpstreamUpdates, this); + m_UpstreamUpdatesThread = std::thread(&UpstreamApplyImpl::ProcessUpstreamUpdates, this); - m_EndpointMonitorThread = std::thread(&DefaultUpstreamApply::MonitorEndpoints, this); + m_EndpointMonitorThread = std::thread(&UpstreamApplyImpl::MonitorEndpoints, this); } return m_RunState.IsRunning; @@ -1558,18 +1628,27 @@ private: ////////////////////////////////////////////////////////////////////////// std::unique_ptr -MakeUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) +UpstreamApply::Create(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore) { - return std::make_unique(Options, CasStore, CidStore); + return std::make_unique(Options, CasStore, CidStore); } std::unique_ptr -MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, - std::unique_ptr TokenProvider, - CasStore& CasStore, - CidStore& CidStore) +UpstreamApplyEndpoint::CreateHordeEndpoint(const CloudCacheClientOptions& ComputeOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& StorageAuthConfig, + CasStore& CasStore, + CidStore& CidStore, + AuthMgr& Mgr) { - return std::make_unique(Options, std::move(TokenProvider), CasStore, CidStore); + return std::make_unique(ComputeOptions, + ComputeAuthConfig, + StorageOptions, + StorageAuthConfig, + CasStore, + CidStore, + Mgr); } } // namespace zen diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h index c56a22ac3..44c08e30e 100644 --- a/zenserver/upstream/upstreamapply.h +++ b/zenserver/upstream/upstreamapply.h @@ -25,6 +25,8 @@ class CidStore; class ZenCacheStore; struct CloudCacheClientOptions; class CloudCacheTokenProvider; +struct UpstreamAuthConfig; +class AuthMgr; enum class UpstreamApplyState : int32_t { @@ -129,10 +131,18 @@ public: virtual PostUpstreamApplyResult PostApply(const UpstreamApplyRecord& ApplyRecord) = 0; virtual GetUpstreamApplyUpdatesResult GetUpdates() = 0; virtual UpstreamApplyEndpointStats& Stats() = 0; + + static std::unique_ptr CreateHordeEndpoint(const CloudCacheClientOptions& ComputeOptions, + const UpstreamAuthConfig& ComputeAuthConfig, + const CloudCacheClientOptions& StorageOptions, + const UpstreamAuthConfig& StorageAuthConfig, + CasStore& CasStore, + CidStore& CidStore, + AuthMgr& Mgr); }; /** - * Manages one or more upstream cache endpoints. + * Manages one or more upstream compute endpoints. */ class UpstreamApply { @@ -157,14 +167,9 @@ public: virtual EnqueueResult EnqueueUpstream(UpstreamApplyRecord ApplyRecord) = 0; virtual StatusResult GetStatus(const IoHash& WorkerId, const IoHash& ActionId) = 0; virtual void GetStatus(CbObjectWriter& CbO) = 0; -}; -std::unique_ptr MakeUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore); - -std::unique_ptr MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, - std::unique_ptr TokenProvider, - CasStore& CasStore, - CidStore& CidStore); + static std::unique_ptr Create(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore); +}; } // namespace zen diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index a684272c4..0d9126334 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -279,15 +279,20 @@ public: #endif #if ZEN_WITH_COMPUTE_SERVICES - ZEN_INFO("instantiating compute services"); + if (ServerOptions.ComputeServiceEnabled) + { + ZEN_INFO("instantiating compute services"); - std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; - zen::CreateDirectories(SandboxDir); - m_HttpLaunchService = std::make_unique(*m_CasStore, SandboxDir); + std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; + zen::CreateDirectories(SandboxDir); + m_HttpLaunchService = std::make_unique(*m_CasStore, SandboxDir); - std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply"; - zen::CreateDirectories(ApplySandboxDir); - m_HttpFunctionService = std::make_unique(*m_CasStore, *m_CidStore, ApplySandboxDir); + InitializeCompute(ServerOptions); + } + else + { + ZEN_INFO("NOT instantiating compute services"); + } #endif // ZEN_WITH_COMPUTE_SERVICES if (ServerOptions.StructuredCacheEnabled) @@ -327,14 +332,14 @@ public: m_Http->RegisterService(m_CasService); #if ZEN_WITH_COMPUTE_SERVICES - if (m_HttpLaunchService) + if (ServerOptions.ComputeServiceEnabled) { m_Http->RegisterService(*m_HttpLaunchService); - } - if (m_HttpFunctionService) - { - m_Http->RegisterService(*m_HttpFunctionService); + if (m_HttpFunctionService != nullptr) + { + m_Http->RegisterService(*m_HttpFunctionService); + } } #endif // ZEN_WITH_COMPUTE_SERVICES @@ -360,6 +365,7 @@ public: void InitializeState(const ZenServerOptions& ServerOptions); void InitializeStructuredCache(const ZenServerOptions& ServerOptions); + void InitializeCompute(const ZenServerOptions& ServerOptions); #if ZEN_ENABLE_MESH void StartMesh(int BasePort) @@ -821,6 +827,57 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) m_Http->RegisterService(*m_UpstreamService); } +void +ZenServer::InitializeCompute(const ZenServerOptions& ServerOptions) +{ + ServerOptions; + const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; + + // Horde compute upstream + if (UpstreamConfig.HordeConfig.Url.empty() == false && UpstreamConfig.HordeConfig.Url.empty() == false) + { + std::string_view EndpointName = UpstreamConfig.HordeConfig.Name.empty() ? "Horde"sv : UpstreamConfig.HordeConfig.Name; + + auto ComputeOptions = + zen::CloudCacheClientOptions{.Name = EndpointName, + .ServiceUrl = UpstreamConfig.HordeConfig.Url, + .ComputeCluster = UpstreamConfig.HordeConfig.Cluster, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds), + .UseLegacyDdc = false}; + + auto ComputeAuthConfig = zen::UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.HordeConfig.OAuthUrl, + .OAuthClientId = UpstreamConfig.HordeConfig.OAuthClientId, + .OAuthClientSecret = UpstreamConfig.HordeConfig.OAuthClientSecret, + .OpenIdProvider = UpstreamConfig.HordeConfig.OpenIdProvider, + .AccessToken = UpstreamConfig.HordeConfig.AccessToken}; + + auto StorageOptions = + zen::CloudCacheClientOptions{.Name = EndpointName, + .ServiceUrl = UpstreamConfig.JupiterConfig.Url, + .BlobStoreNamespace = UpstreamConfig.HordeConfig.Namespace, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}; + + auto StorageAuthConfig = zen::UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.JupiterConfig.OAuthUrl, + .OAuthClientId = UpstreamConfig.JupiterConfig.OAuthClientId, + .OAuthClientSecret = UpstreamConfig.JupiterConfig.OAuthClientSecret, + .OpenIdProvider = UpstreamConfig.JupiterConfig.OpenIdProvider, + .AccessToken = UpstreamConfig.JupiterConfig.AccessToken}; + + std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply"; + zen::CreateDirectories(ApplySandboxDir); + m_HttpFunctionService = std::make_unique(*m_CasStore, + *m_CidStore, + ApplySandboxDir, + ComputeOptions, + StorageOptions, + ComputeAuthConfig, + StorageAuthConfig, + *m_AuthMgr); + } +} + //////////////////////////////////////////////////////////////////////////////// class ZenEntryPoint -- cgit v1.2.3 From d4efa2df30437226903a4e4a9a83fd9b3da3117c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 17 Mar 2022 19:37:10 +0100 Subject: Fix so ZenServer compiles when ZEN_WITH_COMPUTE_SERVICES is disabled --- zenserver/zenserver.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 0d9126334..bed60a82a 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -365,7 +365,9 @@ public: void InitializeState(const ZenServerOptions& ServerOptions); void InitializeStructuredCache(const ZenServerOptions& ServerOptions); +#if ZEN_WITH_COMPUTE_SERVICES void InitializeCompute(const ZenServerOptions& ServerOptions); +#endif #if ZEN_ENABLE_MESH void StartMesh(int BasePort) @@ -827,6 +829,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) m_Http->RegisterService(*m_UpstreamService); } +#if ZEN_WITH_COMPUTE_SERVICES void ZenServer::InitializeCompute(const ZenServerOptions& ServerOptions) { @@ -877,6 +880,7 @@ ZenServer::InitializeCompute(const ZenServerOptions& ServerOptions) *m_AuthMgr); } } +#endif //////////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From 871a39086693f9d42bd179dc4b6548f8e288f5f3 Mon Sep 17 00:00:00 2001 From: Joe Kirchoff Date: Thu, 17 Mar 2022 16:30:56 -0700 Subject: Suppress C4305 in third party includes --- zencore/include/zencore/zencore.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index bd5e5a531..60ebd0f5f 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -72,10 +72,11 @@ #ifndef ZEN_THIRD_PARTY_INCLUDES_START # if ZEN_COMPILER_MSC -# define ZEN_THIRD_PARTY_INCLUDES_START \ - __pragma(warning(push)) __pragma(warning(disable : 4668)) /* use of undefined preprocessor macro */ \ - __pragma(warning(disable : 4267)) /* '=': conversion from 'size_t' to 'US' */ \ - __pragma(warning(disable : 4127)) +# define ZEN_THIRD_PARTY_INCLUDES_START \ + __pragma(warning(push)) __pragma(warning(disable : 4668)) /* use of undefined preprocessor macro */ \ + __pragma(warning(disable : 4305)) /* 'if': truncation from 'uint32' to 'bool' */ \ + __pragma(warning(disable : 4267)) /* '=': conversion from 'size_t' to 'US' */ \ + __pragma(warning(disable : 4127)) /* conditional expression is constant */ # elif ZEN_COMPILER_CLANG # define ZEN_THIRD_PARTY_INCLUDES_START \ _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wundef\"") \ -- cgit v1.2.3