aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPer Larsson <[email protected]>2022-03-19 10:41:08 +0100
committerPer Larsson <[email protected]>2022-03-19 10:41:08 +0100
commit703c252710cdae35a35be700fade144e994777c0 (patch)
tree03268a46ea32553c4777ee30cc617f140665938a
parentMinor cleanup. (diff)
parentSuppress C4305 in third party includes (diff)
downloadzen-703c252710cdae35a35be700fade144e994777c0.tar.xz
zen-703c252710cdae35a35be700fade144e994777c0.zip
Merge branch 'main' into streamapi
-rw-r--r--.clang-format35
-rw-r--r--README.md21
-rw-r--r--scripts/bundle.lua106
-rw-r--r--scripts/formatcode.py19
-rw-r--r--scripts/remote_build.py12
-rw-r--r--zencore/compactbinary.cpp16
-rw-r--r--zencore/include/zencore/iobuffer.h17
-rw-r--r--zencore/include/zencore/logging.h2
-rw-r--r--zencore/include/zencore/zencore.h9
-rw-r--r--zencore/thread.cpp8
-rw-r--r--zenhttp/httpserver.cpp20
-rw-r--r--zenserver/compute/apply.cpp38
-rw-r--r--zenserver/compute/apply.h14
-rw-r--r--zenserver/config.cpp95
-rw-r--r--zenserver/config.h15
-rw-r--r--zenserver/experimental/frontend.cpp119
-rw-r--r--zenserver/experimental/frontend.h24
-rw-r--r--zenserver/frontend/frontend.cpp250
-rw-r--r--zenserver/frontend/frontend.h25
-rw-r--r--zenserver/frontend/html/index.html59
-rw-r--r--zenserver/frontend/zipfs.cpp170
-rw-r--r--zenserver/frontend/zipfs.h24
-rw-r--r--zenserver/upstream/jupiter.cpp16
-rw-r--r--zenserver/upstream/jupiter.h5
-rw-r--r--zenserver/upstream/upstreamapply.cpp295
-rw-r--r--zenserver/upstream/upstreamapply.h21
-rw-r--r--zenserver/xmake.lua2
-rw-r--r--zenserver/zenserver.cpp149
28 files changed, 1212 insertions, 374 deletions
diff --git a/.clang-format b/.clang-format
index f1b47c7d7..4688f60ae 100644
--- a/.clang-format
+++ b/.clang-format
@@ -31,7 +31,7 @@ BraceWrapping:
AfterCaseLabel: true
AfterClass: true
AfterControlStatement: Always
- AfterEnum: false
+ AfterEnum: true
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: false
@@ -66,10 +66,6 @@ DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
-ForEachMacros:
- - foreach
- - Q_FOREACH
- - BOOST_FOREACH
IncludeBlocks: Preserve
IncludeCategories:
- Regex: '^<ext/.*\.h>'
@@ -118,31 +114,7 @@ PointerAlignment: Left
RawStringFormats:
- Language: Cpp
Delimiters:
- - cc
- - CC
- cpp
- - Cpp
- - CPP
- - 'c++'
- - 'C++'
- CanonicalDelimiter: ''
- BasedOnStyle: google
- - Language: TextProto
- Delimiters:
- - pb
- - PB
- - proto
- - PROTO
- EnclosingFunctions:
- - EqualsProto
- - EquivToProto
- - PARSE_PARTIAL_TEXT_PROTO
- - PARSE_TEST_PROTO
- - PARSE_TEXT_PROTO
- - ParseTextOrDie
- - ParseTextProtoOrDie
- - ParseTestProto
- - ParsePartialTestProto
CanonicalDelimiter: ''
BasedOnStyle: google
ReflowComments: true
@@ -169,14 +141,9 @@ SpacesInSquareBrackets: false
SpaceBeforeSquareBrackets: false
BitFieldColonSpacing: Both
Standard: Auto
-StatementMacros:
- - Q_UNUSED
- - QT_REQUIRE_VERSION
TabWidth: 4
UseCRLF: false
UseTab: Always
WhitespaceSensitiveMacros:
- STRINGIZE
- - PP_STRINGIZE
- - BOOST_PP_STRINGIZE
...
diff --git a/README.md b/README.md
index f26f23150..5b85d86f9 100644
--- a/README.md
+++ b/README.md
@@ -96,8 +96,8 @@ Next we need the `xmake` build system. For this we will download and install
date `.deb` files;
```
-wget https://github.com/xmake-io/xmake/releases/download/v2.6.2/xmake-v2.6.2.amd64.deb
-sudo dpkg -i xmake-v2.5.7.amd64.deb
+wget https://github.com/xmake-io/xmake/releases/download/v2.6.4/xmake-v2.6.4.amd64.deb
+sudo dpkg -i xmake-v2.6.4.amd64.deb
xmake --version
```
@@ -136,6 +136,23 @@ Note that the command above to set the build variant to debug is optional. Tests
are only built in debug.The `xmake` flags `-vD` can be useful to diagnose
`xmake` issues.
+### Distribution compatibility
+
+Builds for Linux have a dependency on a modern version of the C++ library that
+supports Zen's use of C++20 and the GCC-11 toolchain used (assuming the above
+build instructions are adhered to). However there is no guarantee that the this
+dependency is met when building on one Linux install and running on another. For
+example, at the time of writing the LTS version of Ubuntu and Debian do not have
+GCC-11's version of libstdc++.so.6 available in their package repositories.
+
+To solve this, `xmake bundle` will bundle the required C++ .so and zenserver
+binary together using AppImage. This can briefly be summarised as create a
+squashfs file system containing zenserver and any C++ shared objects list with
+`lld`. This file system is then concatenated with AppImage's runtime. This is
+implemented in `scripts/bundle_linux.sh`. More details can be found here;
+
+https://github.com/AppImage/AppImageKit
+
## Building on Mac
Building on Mac is very similar to Linux; install xmake, clone vcpkg and Zen and
diff --git a/scripts/bundle.lua b/scripts/bundle.lua
index 90fb13657..01268ab14 100644
--- a/scripts/bundle.lua
+++ b/scripts/bundle.lua
@@ -2,8 +2,15 @@
--------------------------------------------------------------------------------
local function _exec(cmd, ...)
- print("--", cmd, ...)
- local ret = os.execv(cmd, {...})
+ local args = {}
+ for _, arg in pairs({...}) do
+ if arg then
+ table.insert(args, arg)
+ end
+ end
+
+ print("--", cmd, table.unpack(args))
+ local ret = os.execv(cmd, args)
print()
return ret
end
@@ -35,23 +42,52 @@ local function _build(arch, debug, config_args)
end
--------------------------------------------------------------------------------
-local function _zip(zip_path, ...)
+local function _zip(store_only, zip_path, ...)
+ -- Here's the rules; if len(...) is 1 and it is a dir then create a zip with
+ -- archive paths like this;
+ --
+ -- glob(foo/bar/**) -> foo/bar/abc, foo/bar/dir/123 -> zip(abc, dir/123)
+ --
+ -- Otherwise assume ... is file paths and add without leading directories;
+ --
+ -- foo/abc, bar/123 -> zip(abc, 123)
+
+ zip_path = path.absolute(zip_path)
os.tryrm(zip_path)
+ local inputs = {...}
+
+ local source_dir = nil
+ if #inputs == 1 and os.isdir(inputs[1]) then
+ source_dir = inputs[1]
+ end
+
import("detect.tools.find_7z")
local cmd = find_7z()
if cmd then
- -- A ./ prefix makes 7z ignore the directory structure
input_paths = {}
- for _, input_path in ipairs({...}) do
- input_path = path.relative(input_path, ".")
- if input_path:sub(2,2) ~= ":" then
- input_path = "./"..input_path
+ if source_dir then
+ -- Suffixing a directory path with a "/." will have 7z set the path
+ -- for archived files relative to that directory.
+ input_paths = { path.join(source_dir, ".") }
+ else
+ for _, input_path in pairs(inputs) do
+ -- If there is a "/./" anywhere in file paths then 7z drops all
+ -- directory information and just archives the file by name
+ input_path = path.relative(input_path, ".")
+ if input_path:sub(2,2) ~= ":" then
+ input_path = "./"..input_path
+ end
+ table.insert(input_paths, input_path)
end
- table.insert(input_paths, input_path)
end
- local ret = _exec("7z", "a", zip_path, table.unpack(input_paths))
+ compression_level = "-mx9"
+ if store_only then
+ compression_level = "-mx0"
+ end
+
+ local ret = _exec("7z", "a", "-r", compression_level, zip_path, table.unpack(input_paths))
if ret > 0 then
raise("Received error from 7z")
end
@@ -62,10 +98,29 @@ local function _zip(zip_path, ...)
import("detect.tools.find_zip")
cmd = find_zip()
if cmd then
- local ret = _exec("zip", "--junk-paths", zip_path, ...)
+ local input_paths = inputs
+ local cwd = os.curdir()
+ if source_dir then
+ os.cd(source_dir)
+ input_paths = { "." }
+ end
+
+ compression_level = "-9"
+ if store_only then
+ compression_level = "-0"
+ end
+
+ local strip_leading_path = nil
+ if not source_dir then
+ strip_leading_path = "--junk-paths"
+ end
+
+ local ret = _exec("zip", "-r", compression_level, strip_leading_path, zip_path, table.unpack(input_paths))
if ret > 0 then
raise("Received error from zip")
end
+
+ os.cd(cwd)
return
end
print("zip not found")
@@ -90,6 +145,21 @@ local function _find_vcpkg_binary(triple, port, binary)
end
--------------------------------------------------------------------------------
+local function _append_content_zip(bin_path)
+ local zip_path = "build/frontend.zip"
+ local content_dir = "zenserver/frontend/html/"
+ _zip(true, zip_path, content_dir)
+
+ zip_file = io.open(zip_path, "rb")
+ local zip_data = zip_file:read("*all")
+ zip_file:close()
+
+ bin_file = io.open(bin_path, "ab")
+ bin_file:write(zip_data)
+ bin_file:close()
+end
+
+--------------------------------------------------------------------------------
local function main_windows()
import("core.base.option")
@@ -102,12 +172,14 @@ local function main_windows()
_build("x64", false, config_args)
+ _append_content_zip("build/windows/x64/release/zenserver.exe")
+
local crashpad_handler_path = _find_vcpkg_binary(
"x64-windows-static",
"sentry-native",
"crashpad_handler.exe")
- _zip(
+ _zip(false,
zip_path,
"build/windows/x64/release/zenserver.exe",
"build/windows/x64/release/zenserver.pdb",
@@ -129,9 +201,11 @@ local function main_mac()
"build/macosx/arm64/release/zenserver"
)
if ret > 0 then
- raise("Failed creating univeral binary")
+ raise("Failed creating universal binary")
end
+ _append_content_zip("build/macosx/universal/release/zenserver")
+
-- At the time of writing vcpkg does not support sentry-native on arm64. Once
-- it does we can create a univeral binary for this. For now just bundle x64
local crashpad_handler_path = _find_vcpkg_binary(
@@ -140,7 +214,7 @@ local function main_mac()
"crashpad_handler")
-- Zip
- _zip(
+ _zip(false,
"build/zenserver-macos.zip",
"build/macosx/universal/release/zenserver",
crashpad_handler_path)
@@ -159,9 +233,11 @@ local function main_linux()
"crashpad_handler")
--]]
+ _append_content_zip("build/linux/x86_64/release/zenserver")
+
_exec("scripts/bundle_linux.sh")
- _zip(
+ _zip(false,
"build/zenserver-linux.zip",
"build/appimage/zenserver",
crashpad_handler_path)
diff --git a/scripts/formatcode.py b/scripts/formatcode.py
index dc13ae117..49a8753da 100644
--- a/scripts/formatcode.py
+++ b/scripts/formatcode.py
@@ -1,8 +1,9 @@
import argparse
-import os
import fileinput
+import os
import pathlib
import re
+import subprocess
match_expressions = []
valid_extensions = []
@@ -89,6 +90,20 @@ def parse_match_expressions(wildcards, matches):
print(f'Could not parse input --match expression \'{regex}\': {str(ex)}')
quit()
+def validate_clang_format():
+ vstring = subprocess.check_output("clang-format --version", shell=True).decode().rstrip()
+
+ match = re.search(r'(\d+)\.(\d+)(\.(\d+))?$', vstring)
+ if not match:
+ raise ValueError("invalid version number '%s'" % vstring)
+
+ (major, minor, patch) = match.group(1, 2, 4)
+
+ if int(major) < 13:
+ if int(minor) == 0:
+ if int(patch) < 1:
+ raise ValueError(f'invalid clang-format version -- we require at least v12.0.1')
+
def _main():
global root_dir, use_batching
@@ -106,6 +121,8 @@ def _main():
root_dir = pathlib.Path(__file__).parent.parent.resolve()
use_batching = options.use_batching
+ validate_clang_format()
+
while True:
if (os.path.isfile(".clang-format")):
scan_zen(".")
diff --git a/scripts/remote_build.py b/scripts/remote_build.py
index 08e44c9ad..70f9cf9bf 100644
--- a/scripts/remote_build.py
+++ b/scripts/remote_build.py
@@ -106,13 +106,15 @@ def _local(args):
_header("Validating remote host and credentials")
- # Validate key file. OpenSSL needs a trailing EOL, LibreSSL doesn't
+ # Validate key file. Git's SSH uses OpenSSL which needs UNIX line-endings
if args.keyfile:
with open(args.keyfile, "rt") as key_file:
- lines = [x for x in key_file]
- if not lines[-1].endswith("\n"):
- print("!! ERROR: key file must end with a new line")
- return 1
+ lines = [x.strip() for x in key_file]
+
+ with open(args.keyfile, "wb") as key_file:
+ for line in lines:
+ key_file.write(line.encode() + b"\n")
+
identity = ("-i", args.keyfile)
else:
identity = ()
diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp
index 902ec26c8..7cc6db68a 100644
--- a/zencore/compactbinary.cpp
+++ b/zencore/compactbinary.cpp
@@ -1754,7 +1754,7 @@ TEST_CASE("uson")
SUBCASE("CbField(None|Type|Name)")
{
constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName;
- constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'};
+ const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'};
CbFieldView NoneField(NoneBytes);
CHECK(NoneField.GetSize() == sizeof(NoneBytes));
@@ -1775,7 +1775,7 @@ TEST_CASE("uson")
SUBCASE("CbField(None|Type)")
{
constexpr CbFieldType FieldType = CbFieldType::None;
- constexpr const char NoneBytes[] = {char(FieldType)};
+ const char NoneBytes[] = {char(FieldType)};
CbFieldView NoneField(NoneBytes);
CHECK(NoneField.GetSize() == sizeof NoneBytes);
@@ -1792,14 +1792,14 @@ TEST_CASE("uson")
SUBCASE("CbField(None|Name)")
{
constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName;
- constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'};
+ const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'};
CbFieldView NoneField(NoneBytes + 1, FieldType);
CHECK(NoneField.GetSize() == uint64_t(sizeof NoneBytes));
CHECK(NoneField.GetName().compare("Name") == 0);
CHECK(NoneField.HasName() == true);
CHECK(NoneField.HasValue() == false);
CHECK(NoneField.GetHash() == IoHash::HashBuffer(NoneBytes, sizeof NoneBytes));
- CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1));
+ CHECK(NoneField.GetView() == MemoryView(NoneBytes + 1, sizeof NoneBytes - 1));
MemoryView SerializedView;
CHECK(NoneField.TryGetSerializedView(SerializedView) == false);
@@ -1810,15 +1810,15 @@ TEST_CASE("uson")
SUBCASE("CbField(None|EmptyName)")
{
- constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName;
- constexpr const uint8_t NoneBytes[] = {uint8_t(FieldType), 0};
- CbFieldView NoneField(NoneBytes + 1, FieldType);
+ constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName;
+ const uint8_t NoneBytes[] = {uint8_t(FieldType), 0};
+ CbFieldView NoneField(NoneBytes + 1, FieldType);
CHECK(NoneField.GetSize() == sizeof NoneBytes);
CHECK(NoneField.GetName().empty() == true);
CHECK(NoneField.HasName() == true);
CHECK(NoneField.HasValue() == false);
CHECK(NoneField.GetHash() == IoHash::HashBuffer(NoneBytes, sizeof NoneBytes));
- CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1));
+ CHECK(NoneField.GetView() == MemoryView(NoneBytes + 1, sizeof NoneBytes - 1));
MemoryView SerializedView;
CHECK(NoneField.TryGetSerializedView(SerializedView) == false);
}
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index 5dbbb95bf..449236127 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -27,6 +27,10 @@ enum class ZenContentType : uint8_t
kCompressedBinary = 7,
kUnknownContentType = 8,
kHTML = 9,
+ kJavaScript = 10,
+ kCSS = 11,
+ kPNG = 12,
+ kIcon = 13,
kCOUNT
};
@@ -58,6 +62,15 @@ ToString(ZenContentType ContentType)
return "yaml"sv;
case ZenContentType::kHTML:
return "html"sv;
+ case ZenContentType::kJavaScript:
+ return "javascript"sv;
+ case ZenContentType::kCSS:
+ return "css"sv;
+ case ZenContentType::kPNG:
+ return "png"sv;
+ case ZenContentType::kIcon:
+ return "icon"sv;
+
}
}
@@ -336,7 +349,7 @@ public:
/** Create a buffer which references a sequence of bytes inside another buffer
*/
- ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes);
+ ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes=~0ull);
/** Create a buffer which references a range of bytes which we assume will live
* for the entire life time.
@@ -351,7 +364,7 @@ public:
ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize);
ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize);
- inline operator bool() const { return !m_Core->IsNull(); }
+ inline explicit operator bool() const { return !m_Core->IsNull(); }
inline operator MemoryView() const& { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); }
inline void MakeOwned() { return m_Core->MakeOwned(); }
[[nodiscard]] inline bool IsOwned() const { return m_Core->IsOwned(); }
diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h
index 74ab0f81f..7c2deeed7 100644
--- a/zencore/include/zencore/logging.h
+++ b/zencore/include/zencore/logging.h
@@ -54,7 +54,7 @@ struct LogCategory
#define ZEN_DEFINE_LOG_CATEGORY_STATIC(Category, Name) \
static struct LogCategory##Category : public LogCategory \
{ \
- LogCategory##Category() : LogCategory(Name) {} \
+ LogCategory##Category() : LogCategory(Name) {} \
} Category;
#define ZEN_LOG_TRACE(Category, fmtstr, ...) \
diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h
index bd5e5a531..60ebd0f5f 100644
--- a/zencore/include/zencore/zencore.h
+++ b/zencore/include/zencore/zencore.h
@@ -72,10 +72,11 @@
#ifndef ZEN_THIRD_PARTY_INCLUDES_START
# if ZEN_COMPILER_MSC
-# define ZEN_THIRD_PARTY_INCLUDES_START \
- __pragma(warning(push)) __pragma(warning(disable : 4668)) /* use of undefined preprocessor macro */ \
- __pragma(warning(disable : 4267)) /* '=': conversion from 'size_t' to 'US' */ \
- __pragma(warning(disable : 4127))
+# define ZEN_THIRD_PARTY_INCLUDES_START \
+ __pragma(warning(push)) __pragma(warning(disable : 4668)) /* use of undefined preprocessor macro */ \
+ __pragma(warning(disable : 4305)) /* 'if': truncation from 'uint32' to 'bool' */ \
+ __pragma(warning(disable : 4267)) /* '=': conversion from 'size_t' to 'US' */ \
+ __pragma(warning(disable : 4127)) /* conditional expression is constant */
# elif ZEN_COMPILER_CLANG
# define ZEN_THIRD_PARTY_INCLUDES_START \
_Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wundef\"") \
diff --git a/zencore/thread.cpp b/zencore/thread.cpp
index 7b1396f5f..c2ecc8d72 100644
--- a/zencore/thread.cpp
+++ b/zencore/thread.cpp
@@ -75,7 +75,7 @@ SetNameInternal(DWORD thread_id, const char* name)
#endif
#if ZEN_PLATFORM_LINUX
-const bool bNoZombieChildren = [] () {
+const bool bNoZombieChildren = []() {
// When a child process exits it is put into a zombie state until the parent
// collects its result. This doesn't fit the Windows-like model that Zen uses
// where there is a less strict familial model and no zombification. Ignoring
@@ -87,7 +87,7 @@ const bool bNoZombieChildren = [] () {
Action.sa_handler = SIG_IGN;
sigaction(SIGCHLD, &Action, nullptr);
return true;
-} ();
+}();
#endif
void
@@ -647,10 +647,10 @@ ProcessHandle::Wait(int TimeoutMs)
timespec SleepTime = {0, SleepMs * 1000 * 1000};
for (int i = 0;; i += SleepMs)
{
-#if ZEN_PLATFORM_MAC
+# if ZEN_PLATFORM_MAC
int WaitState = 0;
waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED);
-#endif
+# endif
if (kill(m_Pid, 0) < 0)
{
diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp
index c8e11468e..710b6f356 100644
--- a/zenhttp/httpserver.cpp
+++ b/zenhttp/httpserver.cpp
@@ -60,6 +60,18 @@ MapContentTypeToString(HttpContentType ContentType)
case HttpContentType::kHTML:
return "text/html"sv;
+
+ case HttpContentType::kJavaScript:
+ return "application/javascript"sv;
+
+ case HttpContentType::kCSS:
+ return "text/css"sv;
+
+ case HttpContentType::kPNG:
+ return "image/png"sv;
+
+ case HttpContentType::kIcon:
+ return "image/x-icon"sv;
}
}
@@ -76,6 +88,10 @@ static constinit uint32_t HashCompressedBinary = HashStringDjb2("application/x
static constinit uint32_t HashJson = HashStringDjb2("json"sv);
static constinit uint32_t HashYaml = HashStringDjb2("yaml"sv);
static constinit uint32_t HashHtml = HashStringDjb2("text/html"sv);
+static constinit uint32_t HashJavaScript = HashStringDjb2("application/javascript"sv);
+static constinit uint32_t HashCss = HashStringDjb2("text/css"sv);
+static constinit uint32_t HashPng = HashStringDjb2("image/png"sv);
+static constinit uint32_t HashIcon = HashStringDjb2("image/x-icon"sv);
std::once_flag InitContentTypeLookup;
@@ -96,6 +112,10 @@ struct HashedTypeEntry
{HashText, HttpContentType::kText},
{HashCompressedBinary, HttpContentType::kCompressedBinary},
{HashHtml, HttpContentType::kHTML},
+ {HashJavaScript, HttpContentType::kJavaScript},
+ {HashCss, HttpContentType::kCSS},
+ {HashPng, HttpContentType::kPNG},
+ {HashIcon, HttpContentType::kIcon},
// clang-format on
};
diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp
index e4d5697fa..044078aa4 100644
--- a/zenserver/compute/apply.cpp
+++ b/zenserver/compute/apply.cpp
@@ -6,6 +6,7 @@
# include <upstream/jupiter.h>
# include <upstream/upstreamapply.h>
+# include <upstream/upstreamcache.h>
# include <zencore/compactbinary.h>
# include <zencore/compactbinarybuilder.h>
# include <zencore/compactbinarypackage.h>
@@ -173,10 +174,10 @@ SandboxedFunctionJob::GrantNamedObjectAccess(PWSTR ObjectName, SE_OBJECT_TYPE Ob
.grfAccessMode = GRANT_ACCESS,
.grfInheritance = grfInhericance,
.Trustee = {.pMultipleTrustee = nullptr,
- .MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE,
- .TrusteeForm = TRUSTEE_IS_SID,
- .TrusteeType = TRUSTEE_IS_GROUP,
- .ptstrName = (PWSTR)m_AppContainerSid}};
+ .MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE,
+ .TrusteeForm = TRUSTEE_IS_SID,
+ .TrusteeType = TRUSTEE_IS_GROUP,
+ .ptstrName = (PWSTR)m_AppContainerSid}};
PACL OldAcl = nullptr;
@@ -328,24 +329,29 @@ SandboxedFunctionJob::SpawnJob(std::filesystem::path ExePath)
////////////////////////////////////////////////////////////////////////////////
-HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& BaseDir)
+HttpFunctionService::HttpFunctionService(CasStore& Store,
+ CidStore& InCidStore,
+ const std::filesystem::path& BaseDir,
+ const CloudCacheClientOptions& ComputeOptions,
+ const CloudCacheClientOptions& StorageOptions,
+ const UpstreamAuthConfig& ComputeAuthConfig,
+ const UpstreamAuthConfig& StorageAuthConfig,
+ AuthMgr& Mgr)
: m_Log(logging::Get("apply"))
, m_CasStore(Store)
, m_CidStore(InCidStore)
, m_SandboxPath(BaseDir / "scratch")
, m_FunctionPath(BaseDir / "func")
{
- m_UpstreamApply = MakeUpstreamApply({}, m_CasStore, m_CidStore);
-
- CloudCacheAccessToken AccessToken{.Value = "ServiceAccount 0f8056b30bd0df0959be55fc3338159b6f938456d3474aed0087fb965268d079",
- .ExpireTime = CloudCacheAccessToken::TimePoint::max()};
-
- CloudCacheClientOptions Options = {.ServiceUrl = "https://horde.devtools-dev.epicgames.com"sv,
- .DdcNamespace = "default"sv,
- .BlobStoreNamespace = "default"sv};
-
- auto HordeUpstreamEndpoint =
- MakeHordeUpstreamEndpoint(Options, CloudCacheTokenProvider::CreateFromStaticToken(AccessToken), m_CasStore, m_CidStore);
+ m_UpstreamApply = UpstreamApply::Create({}, m_CasStore, m_CidStore);
+
+ auto HordeUpstreamEndpoint = UpstreamApplyEndpoint::CreateHordeEndpoint(ComputeOptions,
+ ComputeAuthConfig,
+ StorageOptions,
+ StorageAuthConfig,
+ m_CasStore,
+ m_CidStore,
+ Mgr);
m_UpstreamApply->RegisterEndpoint(std::move(HordeUpstreamEndpoint));
m_UpstreamApply->Initialize();
diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h
index 161e47e06..e00afcd61 100644
--- a/zenserver/compute/apply.h
+++ b/zenserver/compute/apply.h
@@ -23,6 +23,11 @@ namespace zen {
class CasStore;
class CidStore;
class UpstreamApply;
+class CloudCacheClient;
+class AuthMgr;
+
+struct UpstreamAuthConfig;
+struct CloudCacheClientOptions;
/**
* Lambda style compute function service
@@ -30,7 +35,14 @@ class UpstreamApply;
class HttpFunctionService : public HttpService
{
public:
- HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& SandboxBaseDir);
+ HttpFunctionService(CasStore& Store,
+ CidStore& InCidStore,
+ const std::filesystem::path& BaseDir,
+ const CloudCacheClientOptions& ComputeOptions,
+ const CloudCacheClientOptions& StorageOptions,
+ const UpstreamAuthConfig& ComputeAuthConfig,
+ const UpstreamAuthConfig& StorageAuthConfig,
+ AuthMgr& Mgr);
~HttpFunctionService();
virtual const char* BaseUri() const override;
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index bcacc16c0..adb079d83 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -345,6 +345,62 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"),
"");
+ options.add_option("compute",
+ "",
+ "upstream-horde-url",
+ "URL to a Horde instance.",
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.Url)->default_value(""),
+ "");
+
+ options.add_option("compute",
+ "",
+ "upstream-horde-oauth-url",
+ "URL to the OAuth provier",
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthUrl)->default_value(""),
+ "");
+
+ options.add_option("compute",
+ "",
+ "upstream-horde-oauth-clientid",
+ "The OAuth client ID",
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientId)->default_value(""),
+ "");
+
+ options.add_option("compute",
+ "",
+ "upstream-horde-oauth-clientsecret",
+ "The OAuth client secret",
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientSecret)->default_value(""),
+ "");
+
+ options.add_option("compute",
+ "",
+ "upstream-horde-openid-provider",
+ "Name of a registered Open ID provider",
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.OpenIdProvider)->default_value(""),
+ "");
+
+ options.add_option("compute",
+ "",
+ "upstream-horde-token",
+ "A static authentication token",
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.AccessToken)->default_value(""),
+ "");
+
+ options.add_option("compute",
+ "",
+ "upstream-horde-cluster",
+ "The Horde compute cluster id",
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.Cluster)->default_value(""),
+ "");
+
+ options.add_option("compute",
+ "",
+ "upstream-horde-namespace",
+ "The Jupiter namespace to use with Horde compute",
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.HordeConfig.Namespace)->default_value(""),
+ "");
+
options.add_option("gc",
"",
"gc-enabled",
@@ -596,6 +652,45 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio
}
}
+ if (sol::optional<sol::table> ComputeConfig = lua["compute"])
+ {
+ ServerOptions.ComputeServiceEnabled = ComputeConfig->get_or("enable", ServerOptions.ComputeServiceEnabled);
+
+ if (auto UpstreamConfig = ComputeConfig->get<sol::optional<sol::table>>("upstream"))
+ {
+ if (auto HordeConfig = UpstreamConfig->get<sol::optional<sol::table>>("horde"))
+ {
+ UpdateStringValueFromConfig(HordeConfig.value(),
+ std::string_view("name"),
+ ServerOptions.UpstreamCacheConfig.HordeConfig.Name);
+ UpdateStringValueFromConfig(HordeConfig.value(),
+ std::string_view("url"),
+ ServerOptions.UpstreamCacheConfig.HordeConfig.Url);
+ UpdateStringValueFromConfig(HordeConfig.value(),
+ std::string_view("oauthprovider"),
+ ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthUrl);
+ UpdateStringValueFromConfig(HordeConfig.value(),
+ std::string_view("oauthclientid"),
+ ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientId);
+ UpdateStringValueFromConfig(HordeConfig.value(),
+ std::string_view("oauthclientsecret"),
+ ServerOptions.UpstreamCacheConfig.HordeConfig.OAuthClientSecret);
+ UpdateStringValueFromConfig(HordeConfig.value(),
+ std::string_view("openidprovider"),
+ ServerOptions.UpstreamCacheConfig.HordeConfig.OpenIdProvider);
+ UpdateStringValueFromConfig(HordeConfig.value(),
+ std::string_view("token"),
+ ServerOptions.UpstreamCacheConfig.HordeConfig.AccessToken);
+ UpdateStringValueFromConfig(HordeConfig.value(),
+ std::string_view("cluster"),
+ ServerOptions.UpstreamCacheConfig.HordeConfig.Cluster);
+ UpdateStringValueFromConfig(HordeConfig.value(),
+ std::string_view("namespace"),
+ ServerOptions.UpstreamCacheConfig.HordeConfig.Namespace);
+ };
+ }
+ }
+
if (sol::optional<sol::table> GcConfig = lua["gc"])
{
ServerOptions.GcConfig.IntervalSeconds = GcConfig.value().get_or("intervalseconds", 0);
diff --git a/zenserver/config.h b/zenserver/config.h
index fd569bdb1..a7a7815a8 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -33,6 +33,19 @@ struct ZenUpstreamJupiterConfig
bool UseLegacyDdc = false;
};
+struct ZenUpstreamHordeConfig
+{
+ std::string Name;
+ std::string Url;
+ std::string OAuthUrl;
+ std::string OAuthClientId;
+ std::string OAuthClientSecret;
+ std::string OpenIdProvider;
+ std::string AccessToken;
+ std::string Cluster;
+ std::string Namespace;
+};
+
struct ZenUpstreamZenConfig
{
std::string Name;
@@ -51,6 +64,7 @@ enum class UpstreamCachePolicy : uint8_t
struct ZenUpstreamCacheConfig
{
ZenUpstreamJupiterConfig JupiterConfig;
+ ZenUpstreamHordeConfig HordeConfig;
ZenUpstreamZenConfig ZenConfig;
int32_t UpstreamThreadCount = 4;
int32_t ConnectTimeoutMilliseconds = 5000;
@@ -106,6 +120,7 @@ struct ZenServerOptions
bool IsTest = false;
bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements
bool StructuredCacheEnabled = true;
+ bool ComputeServiceEnabled = true;
bool ShouldCrash = false; // Option for testing crash handling
bool IsFirstRun = false;
bool NoSentry = false;
diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp
deleted file mode 100644
index 4bd3ec90a..000000000
--- a/zenserver/experimental/frontend.cpp
+++ /dev/null
@@ -1,119 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "frontend.h"
-
-#include <zencore/filesystem.h>
-#include <zencore/string.h>
-
-namespace zen {
-
-namespace html {
-
- constexpr std::string_view Index = R"(
-<!DOCTYPE html>
-<html>
-<head>
-<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU" crossorigin="anonymous">
-<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.min.js" integrity="sha384-skAcpIdS7UcVUC05LJ9Dxay8AXcDYfBJqt1CJ85S/CFujBsIzCIv+l9liuYLaMQ/" crossorigin="anonymous"></script>
-<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css">
-<style type="text/css">
-body {
- background-color: #fafafa;
-}
-</style>
-<script type="text/javascript">
- const getCacheStats = () => {
- const opts = { headers: { "Accept": "application/json" } };
- fetch("/stats/z$", opts)
- .then(response => {
- if (!response.ok) {
- throw Error(response.statusText);
- }
- return response.json();
- })
- .then(json => {
- document.getElementById("status").innerHTML = "connected"
- document.getElementById("stats").innerHTML = JSON.stringify(json, null, 4);
- })
- .catch(error => {
- document.getElementById("status").innerHTML = "disconnected"
- document.getElementById("stats").innerHTML = ""
- console.log(error);
- })
- .finally(() => {
- window.setTimeout(getCacheStats, 1000);
- });
- };
- getCacheStats();
-</script>
-</head>
-<body>
- <div class="container">
- <div class="row">
- <div class="text-center mt-5">
- <pre>
-__________ _________ __
-\____ / ____ ____ / _____/_/ |_ ____ _______ ____
- / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \
- / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/
-/_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ >
- \/ \/ \/ \/ \/
- </pre>
- <pre id="status"/>
- </div>
- </div>
- <div class="row">
- <pre class="mb-0">Z$:</pre>
- <pre id="stats"></pre>
- <div>
- </div>
-</body>
-</html>
-)";
-
-} // namespace html
-
-HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory)
-{
-}
-
-HttpFrontendService::~HttpFrontendService()
-{
-}
-
-const char*
-HttpFrontendService::BaseUri() const
-{
- return "/dashboard"; // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp
-}
-
-void
-HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
-{
- using namespace std::literals;
-
- if (m_Directory.empty())
- {
- Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, html::Index);
- }
- else
- {
- std::string_view Uri = Request.RelativeUri();
- std::filesystem::path RelPath{Uri.empty() ? "index.html" : Uri};
- std::filesystem::path AbsPath = m_Directory / RelPath;
-
- FileContents File = ReadFile(AbsPath);
-
- if (!File.ErrorCode)
- {
- // TODO: Map file extension to MIME type
- Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, File.Data[0]);
- }
- else
- {
- return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Ooops!"sv);
- }
- }
-}
-
-} // namespace zen
diff --git a/zenserver/experimental/frontend.h b/zenserver/experimental/frontend.h
deleted file mode 100644
index 2ae20e940..000000000
--- a/zenserver/experimental/frontend.h
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zenhttp/httpserver.h>
-
-#include <filesystem>
-
-namespace zen {
-
-class HttpFrontendService final : public zen::HttpService
-{
-public:
- HttpFrontendService(std::filesystem::path Directory);
- virtual ~HttpFrontendService();
-
- virtual const char* BaseUri() const override;
- virtual void HandleRequest(zen::HttpServerRequest& Request) override;
-
-private:
- std::filesystem::path m_Directory;
-};
-
-} // namespace zen
diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp
new file mode 100644
index 000000000..b87d7e313
--- /dev/null
+++ b/zenserver/frontend/frontend.cpp
@@ -0,0 +1,250 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "frontend.h"
+
+#include <zencore/endian.h>
+#include <zencore/filesystem.h>
+#include <zencore/string.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#if ZEN_PLATFORM_WINDOWS
+# include <Windows.h>
+#endif
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+static IoBuffer FindZipFsInBinary(const IoBuffer& BinBuffer)
+{
+ if (BinBuffer.GetSize() < 4)
+ {
+ return {};
+ }
+
+ uintptr_t Cursor = uintptr_t(BinBuffer.GetData());
+ size_t BinSize = 0;
+
+ uint32_t Magic = *(uint32_t*)(BinBuffer.GetData());
+#if ZEN_PLATFORM_LINUX
+ if (Magic == 0x464c457f)
+ {
+ struct Elf64Header
+ {
+ char Ident[16];
+ uint16_t Type;
+ uint16_t Machine;
+ uint32_t Version;
+ uint64_t Entry;
+ uint64_t ProgHeaderOffset;
+ uint64_t SectionHeaderOffset;
+ uint32_t Flags;
+ uint16_t EhSize;
+ uint16_t ProgHeaderEntrySize;
+ uint16_t ProgHeaderCount;
+ uint16_t SectionHeaderEntrySize;
+ uint16_t SectionHeaderCount;
+ uint16_t SectionStringIndex;
+ };
+
+ struct SectionHeader
+ {
+ uint32_t NameIndex;
+ uint32_t Type;
+ uint64_t Flags;
+ uint64_t Address;
+ uint64_t Offset;
+ uint64_t Size;
+ uint64_t _Other[3];
+ };
+
+ const auto* Elf = (Elf64Header*)Cursor;
+ if (Elf->Ident[4] != 0x02) // Elf64
+ {
+ return {};
+ }
+
+ const auto* Section = (SectionHeader*)(Cursor + Elf->SectionHeaderOffset);
+
+ /*
+ size_t BinSize = 0;
+ for (int i = 0, n = Elf->SectionHeaderCount; i < n; ++i, ++Section)
+ {
+ uint32_t SectionEnd = Section->Offset + Section->Size;
+ BinSize = (SectionEnd > BinSize) ? SectionEnd : BinSize;
+ }
+ */
+
+ // What if the section headers aren't the last thing in the fiile though?
+ BinSize = Elf->SectionHeaderEntrySize;
+ BinSize *= Elf->SectionHeaderCount;
+ BinSize += Elf->SectionHeaderOffset;
+ }
+#elif ZEN_PLATFORM_WINDOWS
+ if ((Magic & 0xffff) == 0x5a4d)
+ {
+ const auto* Dos = (IMAGE_DOS_HEADER*)Cursor;
+ const auto* Nt = (IMAGE_NT_HEADERS64*)(Cursor + Dos->e_lfanew);
+ const auto* Section = (IMAGE_SECTION_HEADER*)(uintptr_t(&Nt->OptionalHeader) + Nt->FileHeader.SizeOfOptionalHeader);
+
+ for (int i = 0, n = Nt->FileHeader.NumberOfSections; i < n; ++i, ++Section)
+ {
+ uint32_t SectionEnd = Section->PointerToRawData + Section->SizeOfRawData;
+ BinSize = (SectionEnd > BinSize) ? SectionEnd : BinSize;
+ }
+ }
+#elif ZEN_PLATFORM_MAC
+ if (Magic == 0xbebafeca)
+ {
+ struct MachInt32
+ {
+ operator uint32_t () const { return ByteSwap(Value); }
+ uint32_t Value;
+ };
+
+ struct MachFatArch
+ {
+ MachInt32 CpuType;
+ MachInt32 SubType;
+ MachInt32 Offset;
+ MachInt32 Size;
+ MachInt32 Alignment;
+ };
+
+ struct MachFatHeader
+ {
+ uint32_t Magic;
+ MachInt32 NumArchs;
+ MachFatArch Archs[];
+ };
+
+ const auto* Header = (MachFatHeader*)Cursor;
+ for (int i = 0, n = Header->NumArchs; i < n; ++i)
+ {
+ const MachFatArch* Arch = Header->Archs + i;
+ uint32_t ArchEnd = Arch->Offset + Arch->Size;
+ BinSize = (ArchEnd > BinSize) ? ArchEnd : BinSize;
+ }
+ }
+#endif // win/linux/mac
+
+ if (!BinSize || BinSize > BinBuffer.GetSize())
+ {
+ return {};
+ }
+
+ return IoBuffer(BinBuffer, BinSize);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+HttpFrontendService::HttpFrontendService(std::filesystem::path Directory)
+: m_Directory(Directory)
+{
+ std::filesystem::path SelfPath = GetRunningExecutablePath();
+
+ // Locate a .zip file appended onto the end of this binary
+ IoBuffer SelfBuffer = IoBufferBuilder::MakeFromFile(SelfPath);
+ IoBuffer SelfTailBuffer = FindZipFsInBinary(SelfBuffer);
+ if (SelfTailBuffer)
+ {
+ m_ZipFs = ZipFs(std::move(SelfTailBuffer));
+ }
+
+#if ZEN_BUILD_DEBUG
+ if (!Directory.empty())
+ {
+ return;
+ }
+
+ std::error_code ErrorCode;
+ for (auto Path = SelfPath.parent_path(); !Path.empty(); Path = Path.parent_path())
+ {
+ if (!std::filesystem::is_regular_file(Path / "xmake.lua", ErrorCode))
+ {
+ continue;
+ }
+
+ auto HtmlDir = (Path / __FILE__).parent_path() / "html";
+ if (std::filesystem::is_directory(HtmlDir, ErrorCode))
+ {
+ m_Directory = HtmlDir;
+ }
+ break;
+ }
+#endif
+}
+
+////////////////////////////////////////////////////////////////////////////////
+HttpFrontendService::~HttpFrontendService()
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+const char*
+HttpFrontendService::BaseUri() const
+{
+ return "/dashboard"; // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp
+}
+
+////////////////////////////////////////////////////////////////////////////////
+void
+HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
+{
+ using namespace std::literals;
+
+ std::string_view Uri = Request.RelativeUri();
+ for (; Uri[0] == '/'; Uri = Uri.substr(1));
+ if (Uri.empty())
+ {
+ Uri = "index.html"sv;
+ }
+
+ // Dismiss if the URI contains .. anywhere to prevent arbitrary file reads
+ if (Uri.find("..") != Uri.npos)
+ {
+ Request.WriteResponse(HttpResponseCode::Forbidden);
+ return;
+ }
+
+ // Map the file extension to a MIME type. To keep things constrained, only a
+ // small subset of file extensions is allowed.
+ HttpContentType ContentType = HttpContentType::kCOUNT;
+ size_t DotIndex = Uri.rfind(".");
+ if (DotIndex != Uri.npos)
+ {
+ const std::string_view DotExt = Uri.substr(DotIndex);
+ if (DotExt == ".html") ContentType = HttpContentType::kHTML;
+ else if (DotExt == ".js") ContentType = HttpContentType::kJSON;
+ else if (DotExt == ".css") ContentType = HttpContentType::kCSS;
+ else if (DotExt == ".png") ContentType = HttpContentType::kPNG;
+ else if (DotExt == ".ico") ContentType = HttpContentType::kIcon;
+ }
+
+ if (ContentType == HttpContentType::kCOUNT)
+ {
+ Request.WriteResponse(HttpResponseCode::Forbidden);
+ return;
+ }
+
+ // The given content directory overrides any zip-fs discovered in the binary
+ if (!m_Directory.empty())
+ {
+ FileContents File = ReadFile(m_Directory / Uri);
+ if (!File.ErrorCode)
+ {
+ Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]);
+ return;
+ }
+ }
+
+ IoBuffer FileBuffer = m_ZipFs.GetFile(Uri);
+ if (FileBuffer)
+ {
+ Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
+ return;
+ }
+
+ Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
+}
+
+} // namespace zen
diff --git a/zenserver/frontend/frontend.h b/zenserver/frontend/frontend.h
new file mode 100644
index 000000000..bf5298169
--- /dev/null
+++ b/zenserver/frontend/frontend.h
@@ -0,0 +1,25 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenhttp/httpserver.h>
+#include "zipfs.h"
+
+#include <filesystem>
+
+namespace zen {
+
+class HttpFrontendService final : public zen::HttpService
+{
+public:
+ HttpFrontendService(std::filesystem::path Directory);
+ virtual ~HttpFrontendService();
+ virtual const char* BaseUri() const override;
+ virtual void HandleRequest(zen::HttpServerRequest& Request) override;
+
+private:
+ ZipFs m_ZipFs;
+ std::filesystem::path m_Directory;
+};
+
+} // namespace zen
diff --git a/zenserver/frontend/html/index.html b/zenserver/frontend/html/index.html
new file mode 100644
index 000000000..252ee621e
--- /dev/null
+++ b/zenserver/frontend/html/index.html
@@ -0,0 +1,59 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU" crossorigin="anonymous">
+ <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.min.js" integrity="sha384-skAcpIdS7UcVUC05LJ9Dxay8AXcDYfBJqt1CJ85S/CFujBsIzCIv+l9liuYLaMQ/" crossorigin="anonymous"></script>
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css">
+ <style type="text/css">
+ body {
+ background-color: #fafafa;
+ }
+ </style>
+ <script type="text/javascript">
+ const getCacheStats = () => {
+ const opts = { headers: { "Accept": "application/json" } };
+ fetch("/stats/z$", opts)
+ .then(response => {
+ if (!response.ok) {
+ throw Error(response.statusText);
+ }
+ return response.json();
+ })
+ .then(json => {
+ document.getElementById("status").innerHTML = "connected"
+ document.getElementById("stats").innerHTML = JSON.stringify(json, null, 4);
+ })
+ .catch(error => {
+ document.getElementById("status").innerHTML = "disconnected"
+ document.getElementById("stats").innerHTML = ""
+ console.log(error);
+ })
+ .finally(() => {
+ window.setTimeout(getCacheStats, 1000);
+ });
+ };
+ getCacheStats();
+ </script>
+</head>
+<body>
+ <div class="container">
+ <div class="row">
+ <div class="text-center mt-5">
+ <pre>
+__________ _________ __
+\____ / ____ ____ / _____/_/ |_ ____ _______ ____
+ / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \
+ / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/
+/_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ >
+ \/ \/ \/ \/ \/
+ </pre>
+ <pre id="status"/>
+ </div>
+ </div>
+ <div class="row">
+ <pre class="mb-0">Z$:</pre>
+ <pre id="stats"></pre>
+ <div>
+ </div>
+</body>
+</html>
diff --git a/zenserver/frontend/zipfs.cpp b/zenserver/frontend/zipfs.cpp
new file mode 100644
index 000000000..5fb9d0177
--- /dev/null
+++ b/zenserver/frontend/zipfs.cpp
@@ -0,0 +1,170 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zipfs.h"
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+namespace {
+
+#if ZEN_COMPILER_MSC
+# pragma warning(push)
+# pragma warning(disable : 4200)
+#endif
+
+using ZipInt16 = uint16_t;
+
+struct ZipInt32
+{
+ operator uint32_t () const { return *(uint32_t*)Parts; }
+ uint16_t Parts[2];
+};
+
+struct EocdRecord
+{
+ enum : uint32_t {
+ Magic = 0x0605'4b50,
+ };
+ ZipInt32 Signature;
+ ZipInt16 ThisDiskIndex;
+ ZipInt16 CdStartDiskIndex;
+ ZipInt16 CdRecordThisDiskCount;
+ ZipInt16 CdRecordCount;
+ ZipInt32 CdSize;
+ ZipInt32 CdOffset;
+ ZipInt16 CommentSize;
+ char Comment[];
+};
+
+struct CentralDirectoryRecord
+{
+ enum : uint32_t {
+ Magic = 0x0201'4b50,
+ };
+
+ ZipInt32 Signature;
+ ZipInt16 VersionMadeBy;
+ ZipInt16 VersionRequired;
+ ZipInt16 Flags;
+ ZipInt16 CompressionMethod;
+ ZipInt16 LastModTime;
+ ZipInt16 LastModDate;
+ ZipInt32 Crc32;
+ ZipInt32 CompressedSize;
+ ZipInt32 OriginalSize;
+ ZipInt16 FileNameLength;
+ ZipInt16 ExtraFieldLength;
+ ZipInt16 CommentLength;
+ ZipInt16 DiskIndex;
+ ZipInt16 InternalFileAttr;
+ ZipInt32 ExternalFileAttr;
+ ZipInt32 Offset;
+ char FileName[];
+};
+
+struct LocalFileHeader
+{
+ enum : uint32_t {
+ Magic = 0x0304'4b50,
+ };
+
+ ZipInt32 Signature;
+ ZipInt16 VersionRequired;
+ ZipInt16 Flags;
+ ZipInt16 CompressionMethod;
+ ZipInt16 LastModTime;
+ ZipInt16 LastModDate;
+ ZipInt32 Crc32;
+ ZipInt32 CompressedSize;
+ ZipInt32 OriginalSize;
+ ZipInt16 FileNameLength;
+ ZipInt16 ExtraFieldLength;
+ char FileName[];
+};
+
+#if ZEN_COMPILER_MSC
+# pragma warning(pop)
+#endif
+
+} // namespace
+
+
+
+//////////////////////////////////////////////////////////////////////////
+ZipFs::ZipFs(IoBuffer&& Buffer)
+{
+ MemoryView View = Buffer.GetView();
+
+ uint8_t* Cursor = (uint8_t*)(View.GetData()) + View.GetSize();
+ if (View.GetSize() < sizeof(EocdRecord))
+ {
+ return;
+ }
+
+ const auto* EocdCursor = (EocdRecord*)(Cursor - sizeof(EocdRecord));
+
+ // It is more correct to search backwards for EocdRecord::Magic as the
+ // comment can be of a variable length. But here we're not going to support
+ // zip files with comments.
+ if (EocdCursor->Signature != EocdRecord::Magic)
+ {
+ return;
+ }
+
+ // Zip64 isn't supported either
+ if (EocdCursor->ThisDiskIndex == 0xffff)
+ {
+ return;
+ }
+
+ Cursor -= View.GetSize();
+
+ const auto* CdCursor = (CentralDirectoryRecord*)(Cursor + EocdCursor->CdOffset);
+ for (int i = 0, n = EocdCursor->CdRecordCount; i < n; ++i)
+ {
+ const CentralDirectoryRecord& Cd = *CdCursor;
+
+ bool Acceptable = true;
+ Acceptable &= (Cd.OriginalSize > 0); // has some content
+ Acceptable &= (Cd.CompressionMethod == 0); // is stored uncomrpessed
+ if (Acceptable)
+ {
+ const uint8_t* Lfh = Cursor + Cd.Offset;
+ if (uintptr_t(Lfh - Cursor) < View.GetSize())
+ {
+ std::string_view FileName(Cd.FileName, Cd.FileNameLength);
+ m_Files.insert(std::make_pair(FileName, FileItem{Lfh, size_t(0)}));
+ }
+ }
+
+ uint32_t ExtraBytes = Cd.FileNameLength + Cd.ExtraFieldLength + Cd.CommentLength;
+ CdCursor = (CentralDirectoryRecord*)(Cd.FileName + ExtraBytes);
+ }
+
+ m_Buffer = std::move(Buffer);
+}
+
+//////////////////////////////////////////////////////////////////////////
+IoBuffer ZipFs::GetFile(const std::string_view& FileName) const
+{
+ FileMap::iterator Iter = m_Files.find(FileName);
+ if (Iter == m_Files.end())
+ {
+ return{};
+ }
+
+ FileItem& Item = Iter->second;
+ if (Item.GetSize() > 0)
+ {
+ return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize());
+ }
+
+ const auto* Lfh = (LocalFileHeader*)(Item.GetData());
+ Item = MemoryView(
+ Lfh->FileName + Lfh->FileNameLength + Lfh->ExtraFieldLength,
+ Lfh->OriginalSize
+ );
+ return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize());
+}
+
+} // namespace zen
diff --git a/zenserver/frontend/zipfs.h b/zenserver/frontend/zipfs.h
new file mode 100644
index 000000000..a304e9ff5
--- /dev/null
+++ b/zenserver/frontend/zipfs.h
@@ -0,0 +1,24 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/iobuffer.h>
+
+#include <unordered_map>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+class ZipFs
+{
+public:
+ ZipFs() = default;
+ ZipFs(IoBuffer&& Buffer);
+ IoBuffer GetFile(const std::string_view& FileName) const;
+
+private:
+ using FileItem = MemoryView;
+ using FileMap = std::unordered_map<std::string_view, FileItem>;
+ FileMap mutable m_Files;
+ IoBuffer m_Buffer;
+};
+
+} // namespace zen
diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp
index 2b064a610..eef1daab0 100644
--- a/zenserver/upstream/jupiter.cpp
+++ b/zenserver/upstream/jupiter.cpp
@@ -577,12 +577,12 @@ CloudCacheSession::ObjectExists(const std::set<IoHash>& Keys)
}
CloudCacheResult
-CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksData)
+CloudCacheSession::PostComputeTasks(IoBuffer TasksData)
{
ZEN_TRACE_CPU("HordeClient::PostComputeTasks");
ExtendableStringBuilder<256> Uri;
- Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId;
+ Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << m_CacheClient->ComputeCluster();
cpr::Session& Session = GetSession();
const CloudCacheAccessToken& AccessToken = GetAccessToken();
@@ -609,8 +609,11 @@ CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksDa
CloudCacheResult
CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t WaitSeconds)
{
+ ZEN_TRACE_CPU("HordeClient::GetComputeUpdates");
+
ExtendableStringBuilder<256> Uri;
- Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << ChannelId << "/updates?wait=" << WaitSeconds;
+ Uri << m_CacheClient->ServiceUrl() << "/api/v1/compute/" << m_CacheClient->ComputeCluster() << "/updates/" << ChannelId
+ << "?wait=" << WaitSeconds;
cpr::Session& Session = GetSession();
const CloudCacheAccessToken& AccessToken = GetAccessToken();
@@ -703,6 +706,8 @@ CloudCacheSession::VerifyAccessToken(long StatusCode)
CloudCacheResult
CloudCacheSession::CacheTypeExists(std::string_view TypeId, const IoHash& Key)
{
+ ZEN_TRACE_CPU("HordeClient::CacheTypeExists");
+
ExtendableStringBuilder<256> Uri;
Uri << m_CacheClient->ServiceUrl() << "/api/v1/" << TypeId << "/" << m_CacheClient->BlobStoreNamespace() << "/" << Key.ToHexString();
@@ -731,6 +736,8 @@ CloudCacheSession::CacheTypeExists(std::string_view TypeId, const IoHash& Key)
CloudCacheExistsResult
CloudCacheSession::CacheTypeExists(std::string_view TypeId, const std::set<IoHash>& Keys)
{
+ ZEN_TRACE_CPU("HordeClient::CacheTypeExists");
+
ExtendableStringBuilder<256> Query;
for (const auto& Key : Keys)
{
@@ -766,7 +773,7 @@ CloudCacheSession::CacheTypeExists(std::string_view TypeId, const std::set<IoHas
{
IoBuffer Buffer = IoBuffer(zen::IoBuffer::Wrap, Response.text.data(), Response.text.size());
const CbObject ExistsResponse = LoadCompactBinaryObject(Buffer);
- for (auto& Item : ExistsResponse["id"sv])
+ for (auto& Item : ExistsResponse["Needs"sv])
{
if (Item.IsHash())
{
@@ -878,6 +885,7 @@ CloudCacheClient::CloudCacheClient(const CloudCacheClientOptions& Options, std::
, m_ServiceUrl(Options.ServiceUrl)
, m_DdcNamespace(Options.DdcNamespace)
, m_BlobStoreNamespace(Options.BlobStoreNamespace)
+, m_ComputeCluster(Options.ComputeCluster)
, m_ConnectTimeout(Options.ConnectTimeout)
, m_Timeout(Options.Timeout)
, m_TokenProvider(std::move(TokenProvider))
diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h
index ddd7ea160..9854e6f1e 100644
--- a/zenserver/upstream/jupiter.h
+++ b/zenserver/upstream/jupiter.h
@@ -116,7 +116,7 @@ public:
CloudCacheExistsResult CompressedBlobExists(const std::set<IoHash>& Keys);
CloudCacheExistsResult ObjectExists(const std::set<IoHash>& Keys);
- CloudCacheResult PostComputeTasks(std::string_view ChannelId, IoBuffer TasksData);
+ CloudCacheResult PostComputeTasks(IoBuffer TasksData);
CloudCacheResult GetComputeUpdates(std::string_view ChannelId, const uint32_t WaitSeconds = 0);
CloudCacheResult GetObjectTree(const IoHash& Key);
@@ -167,6 +167,7 @@ struct CloudCacheClientOptions
std::string_view ServiceUrl;
std::string_view DdcNamespace;
std::string_view BlobStoreNamespace;
+ std::string_view ComputeCluster;
std::chrono::milliseconds ConnectTimeout{5000};
std::chrono::milliseconds Timeout{};
bool UseLegacyDdc = false;
@@ -184,6 +185,7 @@ public:
CloudCacheAccessToken AcquireAccessToken();
std::string_view DdcNamespace() const { return m_DdcNamespace; }
std::string_view BlobStoreNamespace() const { return m_BlobStoreNamespace; }
+ std::string_view ComputeCluster() const { return m_ComputeCluster; }
std::string_view ServiceUrl() const { return m_ServiceUrl; }
spdlog::logger& Logger() { return m_Log; }
@@ -193,6 +195,7 @@ private:
std::string m_ServiceUrl;
std::string m_DdcNamespace;
std::string m_BlobStoreNamespace;
+ std::string m_ComputeCluster;
std::chrono::milliseconds m_ConnectTimeout{};
std::chrono::milliseconds m_Timeout{};
std::unique_ptr<CloudCacheTokenProvider> m_TokenProvider;
diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp
index 63c334265..918697224 100644
--- a/zenserver/upstream/upstreamapply.cpp
+++ b/zenserver/upstream/upstreamapply.cpp
@@ -23,6 +23,9 @@
# include <zenstore/cas.h>
# include <zenstore/cidstore.h>
+# include <auth/authmgr.h>
+# include <upstream/upstreamcache.h>
+
# include "cache/structuredcachestore.h"
# include "diag/logging.h"
@@ -48,17 +51,76 @@ namespace detail {
class HordeUpstreamApplyEndpoint final : public UpstreamApplyEndpoint
{
public:
- HordeUpstreamApplyEndpoint(const CloudCacheClientOptions& Options,
- std::unique_ptr<zen::CloudCacheTokenProvider> TokenProvider,
- CasStore& CasStore,
- CidStore& CidStore)
+ HordeUpstreamApplyEndpoint(const CloudCacheClientOptions& ComputeOptions,
+ const UpstreamAuthConfig& ComputeAuthConfig,
+ const CloudCacheClientOptions& StorageOptions,
+ const UpstreamAuthConfig& StorageAuthConfig,
+ CasStore& CasStore,
+ CidStore& CidStore,
+ AuthMgr& Mgr)
: m_Log(logging::Get("upstream-apply"))
+ , m_AuthMgr(Mgr)
, m_CasStore(CasStore)
, m_CidStore(CidStore)
{
- m_DisplayName = fmt::format("Horde - '{}'", Options.ServiceUrl);
- m_Client = new CloudCacheClient(Options, std::move(TokenProvider));
+ m_DisplayName = fmt::format("{} - '{}'+'{}'", ComputeOptions.Name, ComputeOptions.ServiceUrl, StorageOptions.ServiceUrl);
m_ChannelId = fmt::format("zen-{}", zen::GetSessionIdString());
+
+ {
+ std::unique_ptr<CloudCacheTokenProvider> TokenProvider;
+
+ if (ComputeAuthConfig.OAuthUrl.empty() == false)
+ {
+ TokenProvider =
+ CloudCacheTokenProvider::CreateFromOAuthClientCredentials({.Url = ComputeAuthConfig.OAuthUrl,
+ .ClientId = ComputeAuthConfig.OAuthClientId,
+ .ClientSecret = ComputeAuthConfig.OAuthClientSecret});
+ }
+ else if (ComputeAuthConfig.OpenIdProvider.empty() == false)
+ {
+ TokenProvider =
+ CloudCacheTokenProvider::CreateFromCallback([this, ProviderName = std::string(ComputeAuthConfig.OpenIdProvider)]() {
+ AuthMgr::OpenIdAccessToken Token = m_AuthMgr.GetOpenIdAccessToken(ProviderName);
+ return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime};
+ });
+ }
+ else
+ {
+ CloudCacheAccessToken AccessToken{.Value = std::string(ComputeAuthConfig.AccessToken),
+ .ExpireTime = CloudCacheAccessToken::TimePoint::max()};
+ TokenProvider = CloudCacheTokenProvider::CreateFromStaticToken(AccessToken);
+ }
+
+ m_Client = new CloudCacheClient(ComputeOptions, std::move(TokenProvider));
+ }
+
+ {
+ std::unique_ptr<CloudCacheTokenProvider> TokenProvider;
+
+ if (StorageAuthConfig.OAuthUrl.empty() == false)
+ {
+ TokenProvider =
+ CloudCacheTokenProvider::CreateFromOAuthClientCredentials({.Url = StorageAuthConfig.OAuthUrl,
+ .ClientId = StorageAuthConfig.OAuthClientId,
+ .ClientSecret = StorageAuthConfig.OAuthClientSecret});
+ }
+ else if (StorageAuthConfig.OpenIdProvider.empty() == false)
+ {
+ TokenProvider =
+ CloudCacheTokenProvider::CreateFromCallback([this, ProviderName = std::string(StorageAuthConfig.OpenIdProvider)]() {
+ AuthMgr::OpenIdAccessToken Token = m_AuthMgr.GetOpenIdAccessToken(ProviderName);
+ return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime};
+ });
+ }
+ else
+ {
+ CloudCacheAccessToken AccessToken{.Value = std::string(StorageAuthConfig.AccessToken),
+ .ExpireTime = CloudCacheAccessToken::TimePoint::max()};
+ TokenProvider = CloudCacheTokenProvider::CreateFromStaticToken(AccessToken);
+ }
+
+ m_StorageClient = new CloudCacheClient(StorageOptions, std::move(TokenProvider));
+ }
}
virtual ~HordeUpstreamApplyEndpoint() = default;
@@ -109,10 +171,11 @@ namespace detail {
m_PendingTasks[UpstreamData.TaskId] = ApplyRecord;
}
- CloudCacheSession Session(m_Client);
+ CloudCacheSession ComputeSession(m_Client);
+ CloudCacheSession StorageSession(m_StorageClient);
{
- CloudCacheResult Result = BatchPutBlobsIfMissing(Session, UpstreamData.Blobs);
+ CloudCacheResult Result = BatchPutBlobsIfMissing(StorageSession, UpstreamData.Blobs);
Bytes += Result.Bytes;
ElapsedSeconds += Result.ElapsedSeconds;
if (!Result.Success)
@@ -126,7 +189,7 @@ namespace detail {
}
{
- CloudCacheResult Result = BatchPutObjectsIfMissing(Session, UpstreamData.Objects);
+ CloudCacheResult Result = BatchPutObjectsIfMissing(StorageSession, UpstreamData.Objects);
Bytes += Result.Bytes;
ElapsedSeconds += Result.ElapsedSeconds;
if (!Result.Success)
@@ -136,17 +199,33 @@ namespace detail {
.Bytes = Bytes,
.ElapsedSeconds = ElapsedSeconds};
}
+
+ Result = StorageSession.PutRef("requests"sv,
+ UpstreamData.TaskId,
+ UpstreamData.Objects[UpstreamData.TaskId].GetBuffer().AsIoBuffer(),
+ ZenContentType::kCbObject);
+
+ Bytes += Result.Bytes;
+ ElapsedSeconds += Result.ElapsedSeconds;
+ if (!Result.Success)
+ {
+ return {.Error{.ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1,
+ .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to add task ref"},
+ .Bytes = Bytes,
+ .ElapsedSeconds = ElapsedSeconds};
+ }
UpstreamData.Objects.clear();
}
CbObjectWriter Writer;
+ Writer.AddString("c"sv, m_ChannelId);
Writer.AddObjectAttachment("r"sv, UpstreamData.RequirementsId);
Writer.BeginArray("t"sv);
Writer.AddObjectAttachment(UpstreamData.TaskId);
Writer.EndArray();
IoBuffer TasksData = Writer.Save().GetBuffer().AsIoBuffer();
- CloudCacheResult Result = Session.PostComputeTasks(m_ChannelId, TasksData);
+ CloudCacheResult Result = ComputeSession.PostComputeTasks(TasksData);
Bytes += Result.Bytes;
ElapsedSeconds += Result.ElapsedSeconds;
if (!Result.Success)
@@ -214,8 +293,8 @@ namespace detail {
{
return {.Bytes = Bytes,
.ElapsedSeconds = ElapsedSeconds,
- .ErrorCode = ExistsResult.ErrorCode ? ExistsResult.ErrorCode : -1,
- .Reason = !ExistsResult.Reason.empty() ? std::move(ExistsResult.Reason) : "Failed to put blobs"};
+ .ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1,
+ .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to put blobs"};
}
}
@@ -265,8 +344,8 @@ namespace detail {
{
return {.Bytes = Bytes,
.ElapsedSeconds = ElapsedSeconds,
- .ErrorCode = ExistsResult.ErrorCode ? ExistsResult.ErrorCode : -1,
- .Reason = !ExistsResult.Reason.empty() ? std::move(ExistsResult.Reason) : "Failed to put objects"};
+ .ErrorCode = Result.ErrorCode ? Result.ErrorCode : -1,
+ .Reason = !Result.Reason.empty() ? std::move(Result.Reason) : "Failed to put objects"};
}
}
@@ -330,9 +409,10 @@ namespace detail {
try
{
- CloudCacheSession Session(m_Client);
+ CloudCacheSession ComputeSession(m_Client);
+ CloudCacheSession StorageSession(m_StorageClient);
- CloudCacheResult UpdatesResult = Session.GetComputeUpdates(m_ChannelId);
+ CloudCacheResult UpdatesResult = ComputeSession.GetComputeUpdates(m_ChannelId);
Bytes += UpdatesResult.Bytes;
ElapsedSeconds += UpdatesResult.ElapsedSeconds;
if (!UpdatesResult.Success)
@@ -349,9 +429,6 @@ namespace detail {
CbObject TaskStatus = LoadCompactBinaryObject(UpdatesResult.Response);
- // zen::StringBuilder<4096> ObjStr;
- // zen::CompactBinaryToJson(TaskStatus, ObjStr);
-
for (auto& It : TaskStatus["u"sv])
{
CbObjectView Status = It.AsObjectView();
@@ -366,7 +443,7 @@ namespace detail {
continue;
}
- const IoHash TaskId = Status["h"sv].AsObjectAttachment();
+ const IoHash TaskId = Status["h"sv].AsHash();
IoHash WorkerId;
IoHash ActionId;
@@ -383,7 +460,7 @@ namespace detail {
m_PendingTasks.erase(TaskIt);
}
- GetUpstreamApplyResult Result = ProcessTaskStatus(Status, Session);
+ GetUpstreamApplyResult Result = ProcessTaskStatus(Status, StorageSession);
Bytes += Result.Bytes;
ElapsedSeconds += Result.ElapsedSeconds;
@@ -411,9 +488,11 @@ namespace detail {
CasStore& m_CasStore;
CidStore& m_CidStore;
+ AuthMgr& m_AuthMgr;
spdlog::logger& m_Log;
std::string m_DisplayName;
RefPtr<CloudCacheClient> m_Client;
+ RefPtr<CloudCacheClient> m_StorageClient;
UpstreamApplyEndpointStats m_Stats;
std::atomic_bool m_HealthOk{false};
std::string m_ChannelId;
@@ -453,9 +532,9 @@ namespace detail {
return {.Error{.ErrorCode = -1, .Reason = fmt::format("Task {}", ComputeTaskOutcomeToString(Outcome))}};
}
- const IoHash TaskId = TaskStatus["h"sv].AsObjectAttachment();
+ const IoHash TaskId = TaskStatus["h"sv].AsHash();
const DateTime Time = TaskStatus["t"sv].AsDateTime();
- const IoHash ResultHash = TaskStatus["r"sv].AsObjectAttachment();
+ const IoHash ResultHash = TaskStatus["r"sv].AsHash();
const std::string_view AgentId = TaskStatus["a"sv].AsString();
const std::string_view LeaseId = TaskStatus["l"sv].AsString();
@@ -463,105 +542,96 @@ namespace detail {
double ElapsedSeconds{};
// Get Result object and all Object Attachments + Binary Attachment IDs
- CloudCacheResult ObjectTreeResult = Session.GetObjectTree(ResultHash);
- Bytes += ObjectTreeResult.Bytes;
- ElapsedSeconds += ObjectTreeResult.ElapsedSeconds;
+ CloudCacheResult ObjectRefResult = Session.GetRef("responses"sv, ResultHash, ZenContentType::kCbObject);
+ Bytes += ObjectRefResult.Bytes;
+ ElapsedSeconds += ObjectRefResult.ElapsedSeconds;
- if (!ObjectTreeResult.Success)
+ if (!ObjectRefResult.Success)
{
return {.Error{.ErrorCode = -1, .Reason = "Failed to get result object data"},
.Bytes = Bytes,
.ElapsedSeconds = ElapsedSeconds};
}
- std::map<IoHash, IoBuffer> TreeObjectData;
- std::map<IoHash, IoBuffer> TreeBinaryData;
+ std::vector<IoHash> ObjectsToIterate;
+ std::map<IoHash, IoBuffer> ObjectData;
+ std::map<IoHash, IoBuffer> BinaryData;
- MemoryView ResponseView = ObjectTreeResult.Response;
- while (ResponseView.GetSize() > 0)
- {
- CbFieldView Field = CbFieldView(ResponseView.GetData());
- ResponseView += Field.GetSize();
+ ObjectData[ResultHash] = ObjectRefResult.Response;
+ CbObject Object = LoadCompactBinaryObject(ObjectData[ResultHash]);
+ Object.IterateAttachments([&](CbFieldView Field) {
if (Field.IsObjectAttachment())
{
- const IoHash Hash = Field.AsObjectAttachment();
- Field = CbFieldView(ResponseView.GetData());
- ResponseView += Field.GetSize();
- if (!Field.IsObject()) // No data
+ const IoHash AttachmentHash = Field.AsObjectAttachment();
+ if (!ObjectData.contains(AttachmentHash))
{
- TreeObjectData[Hash] = {};
- continue;
+ ObjectsToIterate.push_back(AttachmentHash);
}
- MemoryView FieldView = Field.AsObjectView().GetView();
-
- TreeObjectData[Hash] = IoBuffer(IoBuffer::Wrap, FieldView.GetData(), FieldView.GetSize());
}
else if (Field.IsBinaryAttachment())
{
- const IoHash Hash = Field.AsBinaryAttachment();
- TreeBinaryData[Hash] = {};
- }
- else // Unknown type
- {
+ const IoHash AttachmentHash = Field.AsBinaryAttachment();
+ BinaryData[AttachmentHash] = {};
}
- }
+ });
- for (auto& It : TreeObjectData)
+ while (!ObjectsToIterate.empty())
{
- if (It.second.GetSize() == 0)
+ const IoHash Hash = ObjectsToIterate.back();
+ ObjectsToIterate.pop_back();
+
+ CloudCacheResult ObjectResult = Session.GetObject(Hash);
+ Bytes += ObjectRefResult.Bytes;
+ ElapsedSeconds += ObjectRefResult.ElapsedSeconds;
+ if (!ObjectResult.Success)
{
- CloudCacheResult ObjectResult = Session.GetObject(It.first);
- Bytes += ObjectTreeResult.Bytes;
- ElapsedSeconds += ObjectTreeResult.ElapsedSeconds;
- if (!ObjectTreeResult.Success)
+ return {.Error{.ErrorCode = ObjectResult.ErrorCode, .Reason = std::move(ObjectResult.Reason)},
+ .Bytes = Bytes,
+ .ElapsedSeconds = ElapsedSeconds};
+ }
+ ObjectData[Hash] = std::move(ObjectResult.Response);
+
+ CbObject IterateObject = LoadCompactBinaryObject(ObjectData[Hash]);
+ IterateObject.IterateAttachments([&](CbFieldView Field) {
+ if (Field.IsObjectAttachment())
{
- return {.Error{.ErrorCode = ObjectResult.ErrorCode, .Reason = std::move(ObjectResult.Reason)},
- .Bytes = Bytes,
- .ElapsedSeconds = ElapsedSeconds};
+ const IoHash AttachmentHash = Field.AsObjectAttachment();
+ if (!ObjectData.contains(AttachmentHash))
+ {
+ ObjectsToIterate.push_back(AttachmentHash);
+ }
}
-
- if (!ObjectResult.Success)
+ else if (Field.IsBinaryAttachment())
{
- return {.Error{.ErrorCode = -1, .Reason = "Failed to get result object attachment data"},
- .Bytes = Bytes,
- .ElapsedSeconds = ElapsedSeconds};
+ const IoHash AttachmentHash = Field.AsBinaryAttachment();
+ BinaryData[AttachmentHash] = {};
}
- It.second = std::move(ObjectResult.Response);
- }
+ });
}
- for (auto& It : TreeBinaryData)
+ // Batch load all binary data
+ for (auto& It : BinaryData)
{
- if (It.second.GetSize() == 0)
+ CloudCacheResult BlobResult = Session.GetBlob(It.first);
+ Bytes += ObjectRefResult.Bytes;
+ ElapsedSeconds += ObjectRefResult.ElapsedSeconds;
+ if (!BlobResult.Success)
{
- CloudCacheResult BlobResult = Session.GetBlob(It.first);
- Bytes += ObjectTreeResult.Bytes;
- ElapsedSeconds += ObjectTreeResult.ElapsedSeconds;
- if (!BlobResult.Success)
- {
- return {.Error{.ErrorCode = BlobResult.ErrorCode, .Reason = std::move(BlobResult.Reason)},
- .Bytes = Bytes,
- .ElapsedSeconds = ElapsedSeconds};
- }
-
- if (!BlobResult.Success)
- {
- return {.Error{.ErrorCode = -1, .Reason = "Failed to get result binary attachment data"},
- .Bytes = Bytes,
- .ElapsedSeconds = ElapsedSeconds};
- }
- It.second = std::move(BlobResult.Response);
+ return {.Error{.ErrorCode = BlobResult.ErrorCode, .Reason = std::move(BlobResult.Reason)},
+ .Bytes = Bytes,
+ .ElapsedSeconds = ElapsedSeconds};
}
+ It.second = std::move(BlobResult.Response);
}
- CbObject ResultObject = LoadCompactBinaryObject(TreeObjectData[ResultHash]);
+ CbObject ResultObject = LoadCompactBinaryObject(ObjectData[ResultHash]);
int32_t ExitCode = ResultObject["e"sv].AsInt32();
IoHash StdOutHash = ResultObject["so"sv].AsBinaryAttachment();
IoHash StdErrHash = ResultObject["se"sv].AsBinaryAttachment();
IoHash OutputHash = ResultObject["o"sv].AsObjectAttachment();
- std::string StdOut = std::string((const char*)TreeBinaryData[StdOutHash].GetData(), TreeBinaryData[StdOutHash].GetSize());
- std::string StdErr = std::string((const char*)TreeBinaryData[StdErrHash].GetData(), TreeBinaryData[StdErrHash].GetSize());
+ std::string StdOut = std::string((const char*)BinaryData[StdOutHash].GetData(), BinaryData[StdOutHash].GetSize());
+ std::string StdErr = std::string((const char*)BinaryData[StdErrHash].GetData(), BinaryData[StdErrHash].GetSize());
if (ExitCode != 0)
{
@@ -572,7 +642,7 @@ namespace detail {
.StdErr = std::move(StdErr)};
}
- CbObject OutputObject = LoadCompactBinaryObject(TreeObjectData[OutputHash]);
+ CbObject OutputObject = LoadCompactBinaryObject(ObjectData[OutputHash]);
// Get build.output
IoHash BuildOutputId;
@@ -583,7 +653,7 @@ namespace detail {
if (FileObject["n"sv].AsString() == "Build.output"sv)
{
BuildOutputId = FileObject["h"sv].AsBinaryAttachment();
- BuildOutput = TreeBinaryData[BuildOutputId];
+ BuildOutput = BinaryData[BuildOutputId];
break;
}
}
@@ -604,7 +674,7 @@ namespace detail {
const CbObjectView DirectoryObject = It.AsObjectView();
if (DirectoryObject["n"sv].AsString() == "Outputs"sv)
{
- OutputDirectoryTree = TreeObjectData[DirectoryObject["h"sv].AsObjectAttachment()];
+ OutputDirectoryTree = ObjectData[DirectoryObject["h"sv].AsObjectAttachment()];
break;
}
}
@@ -636,7 +706,7 @@ namespace detail {
// Hash is the compressed data hash, and how it is stored in Horde
IoHash CompressedId = FileObject["h"sv].AsBinaryAttachment();
- if (!TreeBinaryData.contains(CompressedId))
+ if (!BinaryData.contains(CompressedId))
{
Log().warn("Object attachment chunk not retrieved from Horde {}", CompressedId.ToHexString());
return {.Error{.ErrorCode = -1, .Reason = "Object attachment chunk not retrieved from Horde"},
@@ -659,7 +729,7 @@ namespace detail {
}
const IoHash& CompressedId = CidToCompressedId.at(DecompressedId);
- if (!TreeBinaryData.contains(CompressedId))
+ if (!BinaryData.contains(CompressedId))
{
Log().warn("Missing output {} compressed {} uncompressed",
CompressedId.ToHexString(),
@@ -668,7 +738,7 @@ namespace detail {
return;
}
- CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(TreeBinaryData[CompressedId]));
+ CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(BinaryData[CompressedId]));
if (!AttachmentBuffer)
{
@@ -826,11 +896,11 @@ namespace detail {
int64_t Memory = ApplyRecord.WorkerDescriptor["memory"sv].AsInt64();
bool Exclusive = ApplyRecord.WorkerDescriptor["exclusive"sv].AsBool();
- // TODO: Remove override when Horde accepts the UE style Host Platforms (Win64, Linux, Mac)
std::string Condition = fmt::format("Platform == '{}'", HostPlatform);
if (HostPlatform == "Win64")
{
- Condition += " && Pool == 'Win-RemoteExec'";
+ // TODO
+ // Condition += " && Pool == 'Win-RemoteExec'";
}
std::map<std::string_view, int64_t> Resources;
@@ -1176,10 +1246,10 @@ struct UpstreamApplyStats
//////////////////////////////////////////////////////////////////////////
-class DefaultUpstreamApply final : public UpstreamApply
+class UpstreamApplyImpl final : public UpstreamApply
{
public:
- DefaultUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore)
+ UpstreamApplyImpl(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore)
: m_Log(logging::Get("upstream-apply"))
, m_Options(Options)
, m_CasStore(CasStore)
@@ -1188,7 +1258,7 @@ public:
{
}
- virtual ~DefaultUpstreamApply() { Shutdown(); }
+ virtual ~UpstreamApplyImpl() { Shutdown(); }
virtual bool Initialize() override
{
@@ -1213,12 +1283,12 @@ public:
for (uint32_t Idx = 0; Idx < m_Options.ThreadCount; Idx++)
{
- m_UpstreamThreads.emplace_back(&DefaultUpstreamApply::ProcessUpstreamQueue, this);
+ m_UpstreamThreads.emplace_back(&UpstreamApplyImpl::ProcessUpstreamQueue, this);
}
- m_UpstreamUpdatesThread = std::thread(&DefaultUpstreamApply::ProcessUpstreamUpdates, this);
+ m_UpstreamUpdatesThread = std::thread(&UpstreamApplyImpl::ProcessUpstreamUpdates, this);
- m_EndpointMonitorThread = std::thread(&DefaultUpstreamApply::MonitorEndpoints, this);
+ m_EndpointMonitorThread = std::thread(&UpstreamApplyImpl::MonitorEndpoints, this);
}
return m_RunState.IsRunning;
@@ -1558,18 +1628,27 @@ private:
//////////////////////////////////////////////////////////////////////////
std::unique_ptr<UpstreamApply>
-MakeUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore)
+UpstreamApply::Create(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore)
{
- return std::make_unique<DefaultUpstreamApply>(Options, CasStore, CidStore);
+ return std::make_unique<UpstreamApplyImpl>(Options, CasStore, CidStore);
}
std::unique_ptr<UpstreamApplyEndpoint>
-MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options,
- std::unique_ptr<CloudCacheTokenProvider> TokenProvider,
- CasStore& CasStore,
- CidStore& CidStore)
+UpstreamApplyEndpoint::CreateHordeEndpoint(const CloudCacheClientOptions& ComputeOptions,
+ const UpstreamAuthConfig& ComputeAuthConfig,
+ const CloudCacheClientOptions& StorageOptions,
+ const UpstreamAuthConfig& StorageAuthConfig,
+ CasStore& CasStore,
+ CidStore& CidStore,
+ AuthMgr& Mgr)
{
- return std::make_unique<detail::HordeUpstreamApplyEndpoint>(Options, std::move(TokenProvider), CasStore, CidStore);
+ return std::make_unique<detail::HordeUpstreamApplyEndpoint>(ComputeOptions,
+ ComputeAuthConfig,
+ StorageOptions,
+ StorageAuthConfig,
+ CasStore,
+ CidStore,
+ Mgr);
}
} // namespace zen
diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h
index c56a22ac3..44c08e30e 100644
--- a/zenserver/upstream/upstreamapply.h
+++ b/zenserver/upstream/upstreamapply.h
@@ -25,6 +25,8 @@ class CidStore;
class ZenCacheStore;
struct CloudCacheClientOptions;
class CloudCacheTokenProvider;
+struct UpstreamAuthConfig;
+class AuthMgr;
enum class UpstreamApplyState : int32_t
{
@@ -129,10 +131,18 @@ public:
virtual PostUpstreamApplyResult PostApply(const UpstreamApplyRecord& ApplyRecord) = 0;
virtual GetUpstreamApplyUpdatesResult GetUpdates() = 0;
virtual UpstreamApplyEndpointStats& Stats() = 0;
+
+ static std::unique_ptr<UpstreamApplyEndpoint> CreateHordeEndpoint(const CloudCacheClientOptions& ComputeOptions,
+ const UpstreamAuthConfig& ComputeAuthConfig,
+ const CloudCacheClientOptions& StorageOptions,
+ const UpstreamAuthConfig& StorageAuthConfig,
+ CasStore& CasStore,
+ CidStore& CidStore,
+ AuthMgr& Mgr);
};
/**
- * Manages one or more upstream cache endpoints.
+ * Manages one or more upstream compute endpoints.
*/
class UpstreamApply
{
@@ -157,14 +167,9 @@ public:
virtual EnqueueResult EnqueueUpstream(UpstreamApplyRecord ApplyRecord) = 0;
virtual StatusResult GetStatus(const IoHash& WorkerId, const IoHash& ActionId) = 0;
virtual void GetStatus(CbObjectWriter& CbO) = 0;
-};
-std::unique_ptr<UpstreamApply> MakeUpstreamApply(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore);
-
-std::unique_ptr<UpstreamApplyEndpoint> MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options,
- std::unique_ptr<CloudCacheTokenProvider> TokenProvider,
- CasStore& CasStore,
- CidStore& CidStore);
+ static std::unique_ptr<UpstreamApply> Create(const UpstreamApplyOptions& Options, CasStore& CasStore, CidStore& CidStore);
+};
} // namespace zen
diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua
index ad4f67911..569e3c150 100644
--- a/zenserver/xmake.lua
+++ b/zenserver/xmake.lua
@@ -17,7 +17,7 @@ target("zenserver")
add_ldflags("/MANIFEST:EMBED")
add_ldflags("/LTCG")
else
- del_files("windows/**")
+ remove_files("windows/**")
end
if is_plat("macosx") then
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 3c7f9004d..110800315 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -106,7 +106,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
#include "cache/structuredcachestore.h"
#include "compute/apply.h"
#include "diag/diagsvcs.h"
-#include "experimental/frontend.h"
+#include "frontend/frontend.h"
#include "experimental/usnjournal.h"
#include "monitoring/httpstats.h"
#include "monitoring/httpstatus.h"
@@ -279,15 +279,20 @@ public:
#endif
#if ZEN_WITH_COMPUTE_SERVICES
- ZEN_INFO("instantiating compute services");
+ if (ServerOptions.ComputeServiceEnabled)
+ {
+ ZEN_INFO("instantiating compute services");
- std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox";
- zen::CreateDirectories(SandboxDir);
- m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore, SandboxDir);
+ std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox";
+ zen::CreateDirectories(SandboxDir);
+ m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore, SandboxDir);
- std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply";
- zen::CreateDirectories(ApplySandboxDir);
- m_HttpFunctionService = std::make_unique<zen::HttpFunctionService>(*m_CasStore, *m_CidStore, ApplySandboxDir);
+ InitializeCompute(ServerOptions);
+ }
+ else
+ {
+ ZEN_INFO("NOT instantiating compute services");
+ }
#endif // ZEN_WITH_COMPUTE_SERVICES
if (ServerOptions.StructuredCacheEnabled)
@@ -327,14 +332,14 @@ public:
m_Http->RegisterService(m_CasService);
#if ZEN_WITH_COMPUTE_SERVICES
- if (m_HttpLaunchService)
+ if (ServerOptions.ComputeServiceEnabled)
{
m_Http->RegisterService(*m_HttpLaunchService);
- }
- if (m_HttpFunctionService)
- {
- m_Http->RegisterService(*m_HttpFunctionService);
+ if (m_HttpFunctionService != nullptr)
+ {
+ m_Http->RegisterService(*m_HttpFunctionService);
+ }
}
#endif // ZEN_WITH_COMPUTE_SERVICES
@@ -360,6 +365,9 @@ public:
void InitializeState(const ZenServerOptions& ServerOptions);
void InitializeStructuredCache(const ZenServerOptions& ServerOptions);
+#if ZEN_WITH_COMPUTE_SERVICES
+ void InitializeCompute(const ZenServerOptions& ServerOptions);
+#endif
#if ZEN_ENABLE_MESH
void StartMesh(int BasePort)
@@ -826,6 +834,59 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
}
}
+#if ZEN_WITH_COMPUTE_SERVICES
+void
+ZenServer::InitializeCompute(const ZenServerOptions& ServerOptions)
+{
+ ServerOptions;
+ const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig;
+
+ // Horde compute upstream
+ if (UpstreamConfig.HordeConfig.Url.empty() == false && UpstreamConfig.HordeConfig.Url.empty() == false)
+ {
+ std::string_view EndpointName = UpstreamConfig.HordeConfig.Name.empty() ? "Horde"sv : UpstreamConfig.HordeConfig.Name;
+
+ auto ComputeOptions =
+ zen::CloudCacheClientOptions{.Name = EndpointName,
+ .ServiceUrl = UpstreamConfig.HordeConfig.Url,
+ .ComputeCluster = UpstreamConfig.HordeConfig.Cluster,
+ .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds),
+ .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds),
+ .UseLegacyDdc = false};
+
+ auto ComputeAuthConfig = zen::UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.HordeConfig.OAuthUrl,
+ .OAuthClientId = UpstreamConfig.HordeConfig.OAuthClientId,
+ .OAuthClientSecret = UpstreamConfig.HordeConfig.OAuthClientSecret,
+ .OpenIdProvider = UpstreamConfig.HordeConfig.OpenIdProvider,
+ .AccessToken = UpstreamConfig.HordeConfig.AccessToken};
+
+ auto StorageOptions =
+ zen::CloudCacheClientOptions{.Name = EndpointName,
+ .ServiceUrl = UpstreamConfig.JupiterConfig.Url,
+ .BlobStoreNamespace = UpstreamConfig.HordeConfig.Namespace,
+ .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds),
+ .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)};
+
+ auto StorageAuthConfig = zen::UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.JupiterConfig.OAuthUrl,
+ .OAuthClientId = UpstreamConfig.JupiterConfig.OAuthClientId,
+ .OAuthClientSecret = UpstreamConfig.JupiterConfig.OAuthClientSecret,
+ .OpenIdProvider = UpstreamConfig.JupiterConfig.OpenIdProvider,
+ .AccessToken = UpstreamConfig.JupiterConfig.AccessToken};
+
+ std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply";
+ zen::CreateDirectories(ApplySandboxDir);
+ m_HttpFunctionService = std::make_unique<zen::HttpFunctionService>(*m_CasStore,
+ *m_CidStore,
+ ApplySandboxDir,
+ ComputeOptions,
+ StorageOptions,
+ ComputeAuthConfig,
+ StorageAuthConfig,
+ *m_AuthMgr);
+ }
+}
+#endif
+
////////////////////////////////////////////////////////////////////////////////
class ZenEntryPoint
@@ -845,19 +906,75 @@ ZenEntryPoint::ZenEntryPoint(ZenServerOptions& ServerOptions) : m_ServerOptions(
{
}
+#if ZEN_USE_SENTRY
+static void
+SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata)
+{
+ char LogMessageBuffer[160];
+ std::string LogMessage;
+ const char* MessagePtr = LogMessageBuffer;
+
+ int n = vsnprintf(LogMessageBuffer, sizeof LogMessageBuffer, Message, Args);
+
+ if (n >= int(sizeof LogMessageBuffer))
+ {
+ LogMessage.resize(n + 1);
+
+ n = vsnprintf(LogMessage.data(), LogMessage.size(), Message, Args);
+
+ MessagePtr = LogMessage.c_str();
+ }
+
+ switch (Level)
+ {
+ case SENTRY_LEVEL_DEBUG:
+ ConsoleLog().debug("sentry: {}", MessagePtr);
+ break;
+
+ case SENTRY_LEVEL_INFO:
+ ConsoleLog().info("sentry: {}", MessagePtr);
+ break;
+
+ case SENTRY_LEVEL_WARNING:
+ ConsoleLog().warn("sentry: {}", MessagePtr);
+ break;
+
+ case SENTRY_LEVEL_ERROR:
+ ConsoleLog().error("sentry: {}", MessagePtr);
+ break;
+
+ case SENTRY_LEVEL_FATAL:
+ ConsoleLog().critical("sentry: {}", MessagePtr);
+ break;
+ }
+}
+#endif
+
int
ZenEntryPoint::Run()
{
#if ZEN_USE_SENTRY
+ std::string SentryDatabasePath = PathToUtf8(m_ServerOptions.DataDir / ".sentry-native");
+
if (m_ServerOptions.NoSentry == false)
{
sentry_options_t* SentryOptions = sentry_options_new();
sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284");
- sentry_options_set_database_path(SentryOptions, PathToUtf8(m_ServerOptions.DataDir / ".sentry-native").c_str());
- sentry_init(SentryOptions);
+ sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str());
+ sentry_options_set_logger(SentryOptions, SentryLogFunction, this);
+ // sentry_options_set_debug(SentryOptions, 1);
- auto _ = zen::MakeGuard([] { sentry_close(); });
+ if (int ErrorCode = sentry_init(SentryOptions); ErrorCode == 0)
+ {
+ printf("sentry initialized");
+ }
+ else
+ {
+ printf("sentry_init returned failure!");
+ }
}
+
+ auto _ = zen::MakeGuard([] { sentry_close(); });
#endif
auto& ServerOptions = m_ServerOptions;