diff options
| author | Dan Engelbrecht <[email protected]> | 2025-09-22 11:34:53 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-09-22 11:34:53 +0200 |
| commit | 863a023b974be61f67cff37b668504c29d6c295e (patch) | |
| tree | a060e7ec9a47e4e79ddeace2419005759246923d /src | |
| parent | improve builds download partial logic (#501) (diff) | |
| download | zen-863a023b974be61f67cff37b668504c29d6c295e.tar.xz zen-863a023b974be61f67cff37b668504c29d6c295e.zip | |
fetch cloud oplog (#502)
- Feature: Added `zen oplog-download` command to download the oplog body of a cooked output stored in Cloud DDC
- Oplog source is specified using one of the following options
- `--cloud-url` Cloud artifact URL for oplog
- `--host` Base host to resolve download host from
- `--override-host` Specific host to use without resolve
- `--assume-http2` assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake
- `--namespace` Builds Storage namespace
- `--bucket` Builds Storage bucket
- `--build-id` an Oid in hex form for the source identifier to use
- `--yes` suppress conformation query when doing output of a very large oplog to console
- `--quiet` suppress all non-essential console output
- `--output-path` path to oplog output, extension .json or .cb (compact binary). Default is output to console
- `--system-dir` override default system root path
- Authentication options
- Auth token
- `--access-token` http auth Cloud Storage access token
- `--access-token-env` name of environment variable that holds the Http auth Cloud Storage access token
- `--access-token-path` path to json file that holds the Http auth Cloud Storage access token
- `--oidctoken-exe-path` path to OidcToken executable
- OpenId authentication
- `--openid-provider-name` Open ID provider name
- `--openid-provider-url` Open ID provider url
- `--openid-client-id`Open ID client id
- `--openid-refresh-token` Open ID refresh token
- `--encryption-aes-key` 256 bit AES encryption key for storing OpenID credentials
- `--encryption-aes-iv` 128 bit AES encryption initialization vector for storing OpenID credentials
- OAuth authentication
- `--oauth-url` OAuth provier url
- `--oauth-clientid` OAuth client id
- `--oauth-clientsecret` OAuth client secret
- Bugfix: `zen print` command now properly outputs very large compact binary objects as json to console
Diffstat (limited to 'src')
| -rw-r--r-- | src/zen/authutils.cpp | 239 | ||||
| -rw-r--r-- | src/zen/authutils.h | 50 | ||||
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 409 | ||||
| -rw-r--r-- | src/zen/cmds/builds_cmd.h | 23 | ||||
| -rw-r--r-- | src/zen/cmds/print_cmd.cpp | 10 | ||||
| -rw-r--r-- | src/zen/cmds/projectstore_cmd.cpp | 397 | ||||
| -rw-r--r-- | src/zen/cmds/projectstore_cmd.h | 37 | ||||
| -rw-r--r-- | src/zen/zen.cpp | 2 | ||||
| -rw-r--r-- | src/zenutil/buildstoragecache.cpp | 18 | ||||
| -rw-r--r-- | src/zenutil/include/zenutil/buildstoragecache.h | 9 | ||||
| -rw-r--r-- | src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h | 7 | ||||
| -rw-r--r-- | src/zenutil/include/zenutil/jupiter/jupiterhost.h | 35 | ||||
| -rw-r--r-- | src/zenutil/jupiter/jupiterbuildstorage.cpp | 47 | ||||
| -rw-r--r-- | src/zenutil/jupiter/jupiterhost.cpp | 66 |
14 files changed, 909 insertions, 440 deletions
diff --git a/src/zen/authutils.cpp b/src/zen/authutils.cpp new file mode 100644 index 000000000..558398d79 --- /dev/null +++ b/src/zen/authutils.cpp @@ -0,0 +1,239 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "authutils.h" + +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> + +#include <zenhttp/auth/authmgr.h> +#include <zenhttp/httpclient.h> +#include <zenhttp/httpclientauth.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <json11.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { +using namespace std::literals; + +std::string_view +GetDefaultAccessTokenEnvVariableName() +{ +#if ZEN_PLATFORM_WINDOWS + return "UE-CloudDataCacheAccessToken"sv; +#endif +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + return "UE_CloudDataCacheAccessToken"sv; +#endif +} + +std::string +ReadAccessTokenFromJsonFile(const std::filesystem::path& Path) +{ + if (!IsFile(Path)) + { + throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); + } + IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); + std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize()); + std::string JsonError; + json11::Json TokenInfo = json11::Json::parse(JsonText, JsonError); + if (!JsonError.empty()) + { + throw std::runtime_error(fmt::format("failed parsing json file '{}'. Reason: '{}'", Path, JsonError)); + } + const std::string AuthToken = TokenInfo["Token"].string_value(); + if (AuthToken.empty()) + { + throw std::runtime_error(fmt::format("the json file '{}' does not contain a value for \"Token\"", Path)); + } + return AuthToken; +} + +std::filesystem::path +FindOidcTokenExePath(std::string_view OidcTokenAuthExecutablePath) +{ + if (OidcTokenAuthExecutablePath.empty()) + { + const std::string OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL; + std::filesystem::path OidcTokenPath = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred(); + if (IsFile(OidcTokenPath)) + { + return OidcTokenPath; + } + OidcTokenPath = (std::filesystem::current_path() / OidcExecutableName).make_preferred(); + if (IsFile(OidcTokenPath)) + { + return OidcTokenPath; + } + } + else + { + std::filesystem::path OidcTokenPath = std::filesystem::absolute(StringToPath(OidcTokenAuthExecutablePath)).make_preferred(); + if (IsFile(OidcTokenPath)) + { + return OidcTokenPath; + } + } + return {}; +}; + +void +AuthCommandLineOptions::AddOptions(cxxopts::Options& Ops) +{ + // Direct access token (may expire) + Ops.add_option("auth-token", "", "access-token", "Remote host access token", cxxopts::value(m_AccessToken), "<accesstoken>"); + Ops.add_option("auth-token", + "", + "access-token-env", + "Name of environment variable that holds the remote host access token", + cxxopts::value(m_AccessTokenEnv)->default_value(std::string(GetDefaultAccessTokenEnvVariableName())), + "<envvariable>"); + Ops.add_option("auth-token", + "", + "access-token-path", + "Path to json file that holds the remote host access token", + cxxopts::value(m_AccessTokenPath), + "<filepath>"); + + // Auth manager token encryption + Ops.add_option("security", "", "encryption-aes-key", "256 bit AES encryption key", cxxopts::value<std::string>(m_EncryptionKey), ""); + Ops.add_option("security", + "", + "encryption-aes-iv", + "128 bit AES encryption initialization vector", + cxxopts::value<std::string>(m_EncryptionIV), + ""); + + // OpenId acccess token + Ops.add_option("openid", + "", + "openid-provider-name", + "Open ID provider name", + cxxopts::value<std::string>(m_OpenIdProviderName), + "Default"); + Ops.add_option("openid", "", "openid-provider-url", "Open ID provider url", cxxopts::value<std::string>(m_OpenIdProviderUrl), ""); + Ops.add_option("openid", "", "openid-client-id", "Open ID client id", cxxopts::value<std::string>(m_OpenIdClientId), ""); + Ops.add_option("openid", "", "openid-refresh-token", "Open ID refresh token", cxxopts::value<std::string>(m_OpenIdRefreshToken), ""); + + // OAuth acccess token + Ops.add_option("oauth", "", "oauth-url", "OAuth provier url", cxxopts::value<std::string>(m_OAuthUrl)->default_value(""), ""); + Ops.add_option("oauth", "", "oauth-clientid", "OAuth client id", cxxopts::value<std::string>(m_OAuthClientId)->default_value(""), ""); + Ops.add_option("oauth", + "", + "oauth-clientsecret", + "OAuth client secret", + cxxopts::value<std::string>(m_OAuthClientSecret)->default_value(""), + ""); + Ops.add_option("auth", + "", + "oidctoken-exe-path", + "Path to OidcToken executable", + cxxopts::value<std::string>(m_OidcTokenAuthExecutablePath)->default_value(""), + ""); +}; + +void +AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, + const std::filesystem::path& SystemRootDir, + HttpClientSettings& ClientSettings, + std::string_view HostUrl, + std::unique_ptr<AuthMgr>& Auth, + bool Quiet) +{ + auto CreateAuthMgr = [&]() { + ZEN_ASSERT(!SystemRootDir.empty()); + if (!Auth) + { + if (m_EncryptionKey.empty()) + { + m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; + if (!Quiet) + { + ZEN_CONSOLE_WARN("Using default encryption key"); + } + } + + if (m_EncryptionIV.empty()) + { + m_EncryptionIV = "0123456789abcdef"; + if (!Quiet) + { + ZEN_CONSOLE_WARN("Using default encryption initialization vector"); + } + } + + AuthConfig AuthMgrConfig = {.RootDirectory = SystemRootDir / "auth", + .EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey), + .EncryptionIV = AesIV128Bit::FromString(m_EncryptionIV)}; + if (!AuthMgrConfig.EncryptionKey.IsValid()) + { + throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", m_EncryptionKey), Ops.help()); + } + if (!AuthMgrConfig.EncryptionIV.IsValid()) + { + throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", m_EncryptionIV), Ops.help()); + } + Auth = AuthMgr::Create(AuthMgrConfig); + } + }; + + if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty()) + { + CreateAuthMgr(); + std::string ProviderName = m_OpenIdProviderName.empty() ? "Default" : m_OpenIdProviderName; + Auth->AddOpenIdProvider({.Name = ProviderName, .Url = m_OpenIdProviderUrl, .ClientId = m_OpenIdClientId}); + if (!m_OpenIdRefreshToken.empty()) + { + Auth->AddOpenIdToken({.ProviderName = ProviderName, .RefreshToken = m_OpenIdRefreshToken}); + } + } + + auto GetEnvAccessToken = [](const std::string& AccessTokenEnv) -> std::string { + if (!AccessTokenEnv.empty()) + { + return GetEnvVariable(AccessTokenEnv); + } + return {}; + }; + + if (!m_AccessToken.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(m_AccessToken); + } + else if (!m_AccessTokenPath.empty()) + { + MakeSafeAbsolutePathÍnPlace(m_AccessTokenPath); + std::string ResolvedAccessToken = ReadAccessTokenFromJsonFile(m_AccessTokenPath); + if (!ResolvedAccessToken.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); + } + } + else if (!m_OAuthUrl.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOAuthClientCredentials( + {.Url = m_OAuthUrl, .ClientId = m_OAuthClientId, .ClientSecret = m_OAuthClientSecret}); + } + else if (!m_OpenIdProviderName.empty()) + { + CreateAuthMgr(); + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOpenIdProvider(*Auth, m_OpenIdProviderName); + } + else if (std::string ResolvedAccessToken = GetEnvAccessToken(m_AccessTokenEnv); !ResolvedAccessToken.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); + } + else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, HostUrl, Quiet); + } + + if (!ClientSettings.AccessTokenProvider) + { + CreateAuthMgr(); + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth); + } +} +} // namespace zen diff --git a/src/zen/authutils.h b/src/zen/authutils.h new file mode 100644 index 000000000..f8ab4e03b --- /dev/null +++ b/src/zen/authutils.h @@ -0,0 +1,50 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zen.h" + +namespace zen { + +struct HttpClientSettings; +class AuthMgr; + +struct AuthCommandLineOptions +{ + // Direct access token (may expire) + std::string m_AccessToken; + std::string m_AccessTokenEnv; + std::filesystem::path m_AccessTokenPath; + + // Auth manager token encryption + std::string m_EncryptionKey; // 256 bit AES encryption key + std::string m_EncryptionIV; // 128 bit AES initialization vector + + // OpenId acccess token + std::string m_OpenIdProviderName; + std::string m_OpenIdProviderUrl; + std::string m_OpenIdClientId; + std::string m_OpenIdRefreshToken; + + // OAuth acccess token + std::string m_OAuthUrl; + std::string m_OAuthClientId; + std::string m_OAuthClientSecret; + + std::string m_OidcTokenAuthExecutablePath; + + void AddOptions(cxxopts::Options& Ops); + + void ParseOptions(cxxopts::Options& Ops, + const std::filesystem::path& SystemRootDir, + HttpClientSettings& InOutClientSettings, + std::string_view HostUrl, + std::unique_ptr<AuthMgr>& OutAuthMgr, + bool Quiet); +}; + +std::string ReadAccessTokenFromJsonFile(const std::filesystem::path& Path); +std::string_view GetDefaultAccessTokenEnvVariableName(); +std::filesystem::path FindOidcTokenExePath(std::string_view OidcTokenAuthExecutablePath); + +} // namespace zen diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 1188f7cfd..d9f57959e 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -31,6 +31,7 @@ #include <zenutil/chunkingcontroller.h> #include <zenutil/filebuildstorage.h> #include <zenutil/jupiter/jupiterbuildstorage.h> +#include <zenutil/jupiter/jupiterhost.h> #include <zenutil/jupiter/jupitersession.h> #include <zenutil/parallelwork.h> #include <zenutil/wildcard.h> @@ -39,7 +40,6 @@ #include <signal.h> #include <memory> -#include <regex> ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> @@ -823,72 +823,6 @@ namespace { return CleanWipe; } - bool ParseCloudUrl(std::string_view InUrl, - std::string& OutHost, - std::string& OutNamespace, - std::string& OutBucket, - std::string& OutBuildId) - { - std::string Url(InUrl); - const std::string_view ExtendedApiString = "api/v2/builds/"; - if (auto ApiString = ToLower(Url).find(ExtendedApiString); ApiString != std::string::npos) - { - Url.erase(ApiString, ExtendedApiString.length()); - } - - const std::string ArtifactURLRegExString = R"((http[s]?:\/\/.*?)\/(.*?)\/(.*?)\/(.*))"; - const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript | std::regex::icase); - std::match_results<std::string_view::const_iterator> MatchResults; - std::string_view UrlToParse(Url); - if (regex_match(begin(UrlToParse), end(UrlToParse), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5) - { - auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view { - ZEN_ASSERT(Index < MatchResults.size()); - - const auto& Match = MatchResults[Index]; - - return std::string_view(&*Match.first, Match.second - Match.first); - }; - - const std::string_view Host = GetMatch(1); - const std::string_view Namespace = GetMatch(2); - const std::string_view Bucket = GetMatch(3); - const std::string_view BuildId = GetMatch(4); - - OutHost = Host; - OutNamespace = Namespace; - OutBucket = Bucket; - OutBuildId = BuildId; - return true; - } - else - { - return false; - } - } - - std::string ReadAccessTokenFromFile(const std::filesystem::path& Path) - { - if (!IsFile(Path)) - { - throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); - } - IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); - std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize()); - std::string JsonError; - json11::Json TokenInfo = json11::Json::parse(JsonText, JsonError); - if (!JsonError.empty()) - { - throw std::runtime_error(fmt::format("failed parsing json file '{}'. Reason: '{}'", Path, JsonError)); - } - const std::string AuthToken = TokenInfo["Token"].string_value(); - if (AuthToken.empty()) - { - throw std::runtime_error(fmt::format("the json file '{}' does not contain a value for \"Token\"", Path)); - } - return AuthToken; - } - IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& TempFolderPath, const IoHash& Hash, @@ -10246,81 +10180,9 @@ BuildsCommand::BuildsCommand() "<usesparsefiles>"); }; - auto AddAuthOptions = [this](cxxopts::Options& Ops) { - // Direct access token (may expire) - Ops.add_option("auth-token", - "", - "access-token", - "Cloud/Builds Storage access token", - cxxopts::value(m_AccessToken), - "<accesstoken>"); - Ops.add_option("auth-token", - "", - "access-token-env", - "Name of environment variable that holds the cloud/builds Storage access token", - cxxopts::value(m_AccessTokenEnv)->default_value(DefaultAccessTokenEnvVariableName), - "<envvariable>"); - Ops.add_option("auth-token", - "", - "access-token-path", - "Path to json file that holds the cloud/builds Storage access token", - cxxopts::value(m_AccessTokenPath), - "<filepath>"); - - // Auth manager token encryption - Ops.add_option("security", - "", - "encryption-aes-key", - "256 bit AES encryption key", - cxxopts::value<std::string>(m_EncryptionKey), - ""); - Ops.add_option("security", - "", - "encryption-aes-iv", - "128 bit AES encryption initialization vector", - cxxopts::value<std::string>(m_EncryptionIV), - ""); + auto AddCloudOptions = [this](cxxopts::Options& Ops) { + m_AuthOptions.AddOptions(Ops); - // OpenId acccess token - Ops.add_option("openid", - "", - "openid-provider-name", - "Open ID provider name", - cxxopts::value<std::string>(m_OpenIdProviderName), - "Default"); - Ops.add_option("openid", "", "openid-provider-url", "Open ID provider url", cxxopts::value<std::string>(m_OpenIdProviderUrl), ""); - Ops.add_option("openid", "", "openid-client-id", "Open ID client id", cxxopts::value<std::string>(m_OpenIdClientId), ""); - Ops.add_option("openid", - "", - "openid-refresh-token", - "Open ID refresh token", - cxxopts::value<std::string>(m_OpenIdRefreshToken), - ""); - - // OAuth acccess token - Ops.add_option("oauth", "", "oauth-url", "OAuth provier url", cxxopts::value<std::string>(m_OAuthUrl)->default_value(""), ""); - Ops.add_option("oauth", - "", - "oauth-clientid", - "OAuth client id", - cxxopts::value<std::string>(m_OAuthClientId)->default_value(""), - ""); - Ops.add_option("oauth", - "", - "oauth-clientsecret", - "OAuth client secret", - cxxopts::value<std::string>(m_OAuthClientSecret)->default_value(""), - ""); - Ops.add_option("auth", - "", - "oidctoken-exe-path", - "Path to OidcToken executable", - cxxopts::value<std::string>(m_OidcTokenAuthExecutablePath)->default_value(""), - ""); - }; - - auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) { - AddAuthOptions(Ops); Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>"); Ops.add_option("cloud build", "", @@ -10790,13 +10652,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - std::filesystem::path SystemRootDir; - auto ParseSystemOptions = [&]() { - if (m_SystemRootDir.empty()) - { - m_SystemRootDir = PickDefaultSystemRootDirectory(); - } - MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + auto ParseSystemOptions = [&]() { + if (m_SystemRootDir.empty()) + { + m_SystemRootDir = PickDefaultSystemRootDirectory(); + } + MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); }; ParseSystemOptions(); @@ -10817,7 +10678,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", m_BuildId, m_Url), SubOption->help()); } - if (!ParseCloudUrl(m_Url, m_Host, m_Namespace, m_Bucket, m_BuildId)) + if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, m_BuildId)) { throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOption->help()); @@ -10848,130 +10709,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) MakeSafeAbsolutePathÍnPlace(m_StoragePath); }; - std::unique_ptr<AuthMgr> Auth; - HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 2}; - - auto CreateAuthMgr = [&]() { - ZEN_ASSERT(!m_SystemRootDir.empty()); - if (!Auth) - { - if (m_EncryptionKey.empty()) - { - m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; - if (!IsQuiet) - { - ZEN_CONSOLE_WARN("Using default encryption key"); - } - } - - if (m_EncryptionIV.empty()) - { - m_EncryptionIV = "0123456789abcdef"; - if (!IsQuiet) - { - ZEN_CONSOLE_WARN("Using default encryption initialization vector"); - } - } - - AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth", - .EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey), - .EncryptionIV = AesIV128Bit::FromString(m_EncryptionIV)}; - if (!AuthMgrConfig.EncryptionKey.IsValid()) - { - throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", m_EncryptionKey), SubOption->help()); - } - if (!AuthMgrConfig.EncryptionIV.IsValid()) - { - throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", m_EncryptionIV), SubOption->help()); - } - Auth = AuthMgr::Create(AuthMgrConfig); - } - }; - - auto ParseAuthOptions = [&]() { - if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty()) - { - CreateAuthMgr(); - std::string ProviderName = m_OpenIdProviderName.empty() ? "Default" : m_OpenIdProviderName; - Auth->AddOpenIdProvider({.Name = ProviderName, .Url = m_OpenIdProviderUrl, .ClientId = m_OpenIdClientId}); - if (!m_OpenIdRefreshToken.empty()) - { - Auth->AddOpenIdToken({.ProviderName = ProviderName, .RefreshToken = m_OpenIdRefreshToken}); - } - } - - auto GetEnvAccessToken = [](const std::string& AccessTokenEnv) -> std::string { - if (!AccessTokenEnv.empty()) - { - return GetEnvVariable(AccessTokenEnv); - } - return {}; - }; - - auto FindOidcTokenExePath = [](const std::string& OidcTokenAuthExecutablePath) -> std::filesystem::path { - if (OidcTokenAuthExecutablePath.empty()) - { - const std::string OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL; - std::filesystem::path OidcTokenPath = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred(); - if (IsFile(OidcTokenPath)) - { - return OidcTokenPath; - } - } - else - { - std::filesystem::path OidcTokenPath = std::filesystem::absolute(StringToPath(OidcTokenAuthExecutablePath)).make_preferred(); - if (IsFile(OidcTokenPath)) - { - return OidcTokenPath; - } - } - return {}; - }; - - if (!m_AccessToken.empty()) - { - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(m_AccessToken); - } - else if (!m_AccessTokenPath.empty()) - { - MakeSafeAbsolutePathÍnPlace(m_AccessTokenPath); - std::string ResolvedAccessToken = ReadAccessTokenFromFile(m_AccessTokenPath); - if (!ResolvedAccessToken.empty()) - { - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); - } - } - else if (!m_OAuthUrl.empty()) - { - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOAuthClientCredentials( - {.Url = m_OAuthUrl, .ClientId = m_OAuthClientId, .ClientSecret = m_OAuthClientSecret}); - } - else if (!m_OpenIdProviderName.empty()) - { - CreateAuthMgr(); - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOpenIdProvider(*Auth, m_OpenIdProviderName); - } - else if (std::string ResolvedAccessToken = GetEnvAccessToken(m_AccessTokenEnv); !ResolvedAccessToken.empty()) - { - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); - } - else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty()) - { - const std::string& CloudHost = m_OverrideHost.empty() ? m_Host : m_OverrideHost; - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost, IsQuiet); - } - - if (!ClientSettings.AccessTokenProvider) - { - CreateAuthMgr(); - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth); - } - }; - auto ParseOutputOptions = [&]() { if (m_Verbose && m_Quiet) { @@ -11010,6 +10747,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; ParseOutputOptions(); + std::unique_ptr<AuthMgr> Auth; + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 2}; + auto CreateBuildStorage = [&](BuildStorage::Statistics& StorageStats, BuildStorageCache::Statistics& StorageCacheStats, const std::filesystem::path& TempPath, @@ -11027,123 +10770,73 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_Host.empty() || !m_OverrideHost.empty()) { - ParseAuthOptions(); + m_AuthOptions + .ParseOptions(*SubOption, m_SystemRootDir, ClientSettings, m_OverrideHost.empty() ? m_Host : m_OverrideHost, Auth, IsQuiet); } std::string CloudHost; - auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair<bool, std::string> { - HttpClientSettings TestClientSettings{.LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{1000}, - .Timeout = std::chrono::milliseconds{2000}, - .AssumeHttp2 = AssumeHttp2, - .AllowResume = true, - .RetryCount = 0}; - HttpClient TestHttpClient(BaseUrl, TestClientSettings); - HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); - if (TestResponse.IsSuccess()) - { - return {true, ""}; - } - return {false, TestResponse.ErrorMessage("")}; - }; - if (!m_Host.empty()) { if (m_OverrideHost.empty() || m_ZenCacheHost.empty()) { - HttpClient DiscoveryHttpClient(m_Host, ClientSettings); - HttpClient::Response ServerInfoResponse = - DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON)); - if (!ServerInfoResponse.IsSuccess()) - { - ServerInfoResponse.ThrowError(fmt::format("Failed to get list of servers from discovery url '{}'", m_Host)); - } - - std::string_view JsonResponse = ServerInfoResponse.AsText(); - CbObject ResponseObjectView = LoadCompactBinaryFromJson(JsonResponse).AsObject(); - - auto TestHostEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair<bool, std::string> { - HttpClientSettings TestClientSettings{.LogCategory = "httpbuildsclient", - .ConnectTimeout = std::chrono::milliseconds{1000}, - .Timeout = std::chrono::milliseconds{2000}, - .AssumeHttp2 = AssumeHttp2, - .AllowResume = true, - .RetryCount = 0}; - - HttpClient TestHttpClient(BaseUrl, TestClientSettings); - HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); - if (TestResponse.IsSuccess()) - { - return {true, ""}; - } - return {false, TestResponse.ErrorMessage("")}; - }; + JupiterServerDiscovery Response = DiscoverJupiterEndpoints(m_Host, ClientSettings); if (m_OverrideHost.empty()) { - CbArrayView ServerEndpointsArray = ResponseObjectView["serverEndpoints"sv].AsArrayView(); - std::uint64_t ServerCount = ServerEndpointsArray.Num(); - if (ServerCount == 0) + if (Response.ServerEndPoints.empty()) { throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host)); } - for (CbFieldView ServerEndpointView : ServerEndpointsArray) + for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : Response.ServerEndPoints) { - CbObjectView ServerEndpointObject = ServerEndpointView.AsObjectView(); - - std::string_view BaseUrl = ServerEndpointObject["baseUrl"sv].AsString(); - if (!BaseUrl.empty()) + if (!ServerEndpoint.BaseUrl.empty()) { - const bool AssumeHttp2 = ServerEndpointObject["assumeHttp2"sv].AsBool(false); - std::string_view Name = ServerEndpointObject["name"sv].AsString(); - if (auto TestResult = TestHostEndpoint(BaseUrl, AssumeHttp2); TestResult.first) + if (JupiterEndpointTestResult TestResult = + TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2); + TestResult.Success) { - CloudHost = BaseUrl; - m_AssumeHttp2 = AssumeHttp2; - BuildStorageName = Name; + CloudHost = ServerEndpoint.BaseUrl; + m_AssumeHttp2 = ServerEndpoint.AssumeHttp2; + BuildStorageName = ServerEndpoint.Name; break; } else { - ZEN_DEBUG("Unable to reach host {}. Reason: {}", BaseUrl, TestResult.second); + ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason); } } } if (CloudHost.empty()) { - throw std::runtime_error( - fmt::format("Failed to find any usable builds hosts out of {} using {}", ServerCount, m_Host)); + throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}", + Response.ServerEndPoints.size(), + m_Host)); } } - else if (auto TestResult = TestHostEndpoint(m_OverrideHost, m_AssumeHttp2); TestResult.first) + else if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(m_OverrideHost, m_AssumeHttp2); TestResult.Success) { CloudHost = m_OverrideHost; } else { - throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.second)); + throw std::runtime_error( + fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.FailureReason)); } if (m_ZenCacheHost.empty()) { - CbArrayView CacheEndpointsArray = ResponseObjectView["cacheEndpoints"sv].AsArrayView(); - std::uint64_t CacheCount = CacheEndpointsArray.Num(); - for (CbFieldView CacheEndpointView : CacheEndpointsArray) + for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : Response.CacheEndPoints) { - CbObjectView CacheEndpointObject = CacheEndpointView.AsObjectView(); - - std::string_view BaseUrl = CacheEndpointObject["baseUrl"sv].AsString(); - if (!BaseUrl.empty()) + if (!CacheEndpoint.BaseUrl.empty()) { - const bool AssumeHttp2 = CacheEndpointObject["assumeHttp2"sv].AsBool(false); - std::string_view Name = CacheEndpointObject["name"sv].AsString(); - - if (auto TestResult = TestCacheEndpoint(BaseUrl, AssumeHttp2); TestResult.first) + if (ZenCacheEndpointTestResult TestResult = + TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2); + TestResult.Success) { - m_ZenCacheHost = BaseUrl; - CacheAssumeHttp2 = AssumeHttp2; - BuildCacheName = Name; + m_ZenCacheHost = CacheEndpoint.BaseUrl; + CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; + BuildCacheName = CacheEndpoint.Name; break; } } @@ -11158,7 +10851,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); - if (auto TestResult = TestCacheEndpoint(ZenServerLocalHostUrl, false); TestResult.first) + if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenServerLocalHostUrl, false); + TestResult.Success) { m_ZenCacheHost = ZenServerLocalHostUrl; CacheAssumeHttp2 = false; @@ -11169,11 +10863,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } if (m_ZenCacheHost.empty() && !IsQuiet) { - ZEN_CONSOLE_WARN("Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host); + ZEN_CONSOLE_WARN("Failed to find any usable cache hosts out of {} using {}", + Response.CacheEndPoints.size(), + m_Host); } } } - else if (auto TestResult = TestCacheEndpoint(m_ZenCacheHost, false); TestResult.first) + else if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, false); TestResult.Success) { std::string::size_type HostnameStart = 0; std::string::size_type HostnameLength = std::string::npos; @@ -11189,7 +10885,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.second); + ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.FailureReason); m_ZenCacheHost = ""; } } @@ -11233,7 +10929,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } if (!m_ZenCacheHost.empty()) { - if (auto TestResult = TestCacheEndpoint(m_ZenCacheHost, false); TestResult.first) + if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, false); TestResult.Success) { Result.CacheHttp = std::make_unique<HttpClient>(m_ZenCacheHost, HttpClientSettings{.LogCategory = "httpcacheclient", @@ -11265,7 +10961,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.second); + ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.FailureReason); + m_ZenCacheHost = ""; } } if (!IsQuiet) diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index be0422210..27a178c6f 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -2,6 +2,7 @@ #pragma once +#include "../authutils.h" #include "../zen.h" #include <zenhttp/auth/authmgr.h> @@ -67,27 +68,7 @@ private: std::string m_AllowPartialBlockRequests = "mixed"; std::string m_ManifestPath; // Not a std::filesystem::path since it can be relative to m_Path - // Direct access token (may expire) - std::string m_AccessToken; - std::string m_AccessTokenEnv; - std::filesystem::path m_AccessTokenPath; - - // Auth manager token encryption - std::string m_EncryptionKey; // 256 bit AES encryption key - std::string m_EncryptionIV; // 128 bit AES initialization vector - - // OpenId acccess token - std::string m_OpenIdProviderName; - std::string m_OpenIdProviderUrl; - std::string m_OpenIdClientId; - std::string m_OpenIdRefreshToken; - - // OAuth acccess token - std::string m_OAuthUrl; - std::string m_OAuthClientId; - std::string m_OAuthClientSecret; - - std::string m_OidcTokenAuthExecutablePath; + AuthCommandLineOptions m_AuthOptions; std::string m_Verb; // list, upload, download diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp index c3c11a0ea..557808ba7 100644 --- a/src/zen/cmds/print_cmd.cpp +++ b/src/zen/cmds/print_cmd.cpp @@ -19,7 +19,10 @@ PrintCbObject(CbObject Object, bool AddTypeComment) { ExtendableStringBuilder<1024> ObjStr; CompactBinaryToJson(Object, ObjStr, AddTypeComment); - ZEN_CONSOLE("{}", ObjStr); + ForEachStrTok(ObjStr.ToView(), '\n', [](std::string_view Row) { + ZEN_CONSOLE("{}", Row); + return true; + }); } static void @@ -27,7 +30,10 @@ PrintCompactBinary(IoBuffer Data, bool AddTypeComment) { ExtendableStringBuilder<1024> StreamString; CompactBinaryToJson(Data.GetView(), StreamString, AddTypeComment); - ZEN_CONSOLE("{}", StreamString); + ForEachStrTok(StreamString.ToView(), '\n', [](std::string_view Row) { + ZEN_CONSOLE("{}", Row); + return true; + }); } static CbValidateError diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 8ed52c764..b738d16aa 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -4,24 +4,33 @@ #include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryutil.h> #include <zencore/compress.h> +#include <zencore/config.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> +#include <zencore/process.h> #include <zencore/scopeguard.h> #include <zencore/stream.h> #include <zencore/timer.h> #include <zencore/workthreadpool.h> +#include <zenhttp/auth/authmgr.h> #include <zenhttp/formatters.h> #include <zenhttp/httpclient.h> +#include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> +#include <zenutil/jupiter/jupiterbuildstorage.h> +#include <zenutil/jupiter/jupiterhost.h> + ZEN_THIRD_PARTY_INCLUDES_START #include <cpr/cpr.h> #include <json11.hpp> ZEN_THIRD_PARTY_INCLUDES_END #include <signal.h> +#include <iostream> namespace zen { @@ -29,64 +38,7 @@ namespace { using namespace std::literals; - const std::string DefaultJupiterAccessTokenEnvVariableName( -#if ZEN_PLATFORM_WINDOWS - "UE-CloudDataCacheAccessToken"sv -#endif -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - "UE_CloudDataCacheAccessToken"sv -#endif - - ); - - std::string ReadJupiterAccessTokenFromFile(const std::filesystem::path& Path) - { - if (!IsFile(Path)) - { - throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); - } - IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); - std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize()); - std::string JsonError; - json11::Json TokenInfo = json11::Json::parse(JsonText, JsonError); - if (!JsonError.empty()) - { - throw std::runtime_error(fmt::format("failed parsing json file '{}'. Reason: '{}'", Path, JsonError)); - } - const std::string AuthToken = TokenInfo["Token"].string_value(); - if (AuthToken.empty()) - { - throw std::runtime_error(fmt::format("the json file '{}' does not contain a value for \"Token\"", Path)); - } - return AuthToken; - } - - std::filesystem::path FindOidcTokenExePath(std::string_view OidcTokenAuthExecutablePath) - { - if (OidcTokenAuthExecutablePath.empty()) - { - const std::string OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL; - std::filesystem::path OidcTokenPath = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred(); - if (IsFile(OidcTokenPath)) - { - return OidcTokenPath; - } - OidcTokenPath = (std::filesystem::current_path() / OidcExecutableName).make_preferred(); - if (IsFile(OidcTokenPath)) - { - return OidcTokenPath; - } - } - else - { - std::filesystem::path OidcTokenPath = std::filesystem::absolute(StringToPath(OidcTokenAuthExecutablePath)).make_preferred(); - if (IsFile(OidcTokenPath)) - { - return OidcTokenPath; - } - } - return {}; - }; +#define ZEN_CLOUD_STORAGE "Cloud Storage" void WriteAuthOptions(CbObjectWriter& Writer, std::string_view JupiterOpenIdProvider, @@ -105,7 +57,7 @@ namespace { } if (!JupiterAccessTokenPath.empty()) { - std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(JupiterAccessTokenPath); + std::string ResolvedCloudAccessToken = ReadAccessTokenFromJsonFile(JupiterAccessTokenPath); if (!ResolvedCloudAccessToken.empty()) { Writer.AddString("access-token"sv, ResolvedCloudAccessToken); @@ -910,7 +862,7 @@ ExportOplogCommand::ExportOplogCommand() "", "access-token-env", "Name of environment variable that holds the cloud/builds Storage access token", - cxxopts::value(m_JupiterAccessTokenEnv)->default_value(DefaultJupiterAccessTokenEnvVariableName), + cxxopts::value(m_JupiterAccessTokenEnv)->default_value(std::string(GetDefaultAccessTokenEnvVariableName())), "<envvariable>"); m_Options.add_option("", "", @@ -1387,7 +1339,7 @@ ImportOplogCommand::ImportOplogCommand() "", "access-token-env", "Name of environment variable that holds the cloud/builds Storage access token", - cxxopts::value(m_JupiterAccessTokenEnv)->default_value(DefaultJupiterAccessTokenEnvVariableName), + cxxopts::value(m_JupiterAccessTokenEnv)->default_value(std::string(GetDefaultAccessTokenEnvVariableName())), "<envvariable>"); m_Options.add_option("", "", @@ -2236,4 +2188,327 @@ OplogValidateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } } +//////////////////////////// + +OplogDownloadCommand::OplogDownloadCommand() +{ + m_Options.add_options()("h,help", "Print help"); + + m_Options.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>"); + + m_Options.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>"); + m_Options.add_option("", "y", "yes", "Don't query for confirmation", cxxopts::value(m_Yes), "<yes>"); + + auto AddCloudOptions = [this](cxxopts::Options& Ops) { + m_AuthOptions.AddOptions(Ops); + + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>"); + Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>"); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>"); + Ops.add_option("cloud build", + "", + "assume-http2", + "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", + cxxopts::value(m_AssumeHttp2), + "<assumehttp2>"); + + Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>"); + Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>"); + }; + + AddCloudOptions(m_Options); + + m_Options.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + m_Options.add_option("", + "", + "output-path", + "Path to oplog output, extension .json or .cb (compact binary). Default is output to console", + cxxopts::value(m_OutputPath), + "<path>"); + + m_Options.parse_positional({"cloud-url", "output-path"}); + m_Options.positional_help("[<cloud-url> <output-path>]"); +} + +OplogDownloadCommand::~OplogDownloadCommand() +{ +} + +void +OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return; + } + + if (!m_Quiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + } + + auto ParseSystemOptions = [&]() { + if (m_SystemRootDir.empty()) + { + m_SystemRootDir = PickDefaultSystemRootDirectory(); + } + MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + }; + ParseSystemOptions(); + + auto ParseStorageOptions = [&](bool RequireNamespace, bool RequireBucket) { + if (!m_Url.empty()) + { + if (!m_Host.empty()) + { + throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), m_Options.help()); + } + if (!m_Bucket.empty()) + { + throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), + m_Options.help()); + } + if (!m_BuildId.empty()) + { + throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", m_BuildId, m_Url), + m_Options.help()); + } + if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, m_BuildId)) + { + throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", + m_Options.help()); + } + } + + if (!m_OverrideHost.empty() || !m_Host.empty()) + { + if (RequireNamespace && m_Namespace.empty()) + { + throw OptionParseException("'--namespace' is required", m_Options.help()); + } + if (RequireBucket && m_Bucket.empty()) + { + throw OptionParseException("'--bucket' is required", m_Options.help()); + } + } + + if (m_OverrideHost.empty() && m_Host.empty()) + { + throw OptionParseException("'--host' or '--overridehost' is required", m_Options.help()); + } + }; + + std::unique_ptr<AuthMgr> Auth; + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 2}; + + ParseStorageOptions(/*RequireNamespace*/ true, /*RequireBucket*/ true); + + m_AuthOptions.ParseOptions(m_Options, m_SystemRootDir, ClientSettings, m_OverrideHost.empty() ? m_Host : m_OverrideHost, Auth, m_Quiet); + + std::string BuildStorageName = ZEN_CLOUD_STORAGE; + + std::string CloudHost; + + auto TestHostEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair<bool, std::string> { + HttpClientSettings TestClientSettings{.LogCategory = "httpbuildsclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); + if (TestResponse.IsSuccess()) + { + return {true, ""}; + } + return {false, TestResponse.ErrorMessage("")}; + }; + + if (m_OverrideHost.empty()) + { + JupiterServerDiscovery Response = DiscoverJupiterEndpoints(m_Host, ClientSettings); + + if (Response.ServerEndPoints.empty()) + { + throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host)); + } + for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : Response.ServerEndPoints) + { + if (!ServerEndpoint.BaseUrl.empty()) + { + if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2); + TestResult.Success) + { + CloudHost = ServerEndpoint.BaseUrl; + m_AssumeHttp2 = ServerEndpoint.AssumeHttp2; + BuildStorageName = ServerEndpoint.Name; + break; + } + else + { + ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason); + } + } + } + if (CloudHost.empty()) + { + throw std::runtime_error( + fmt::format("Failed to find any usable builds hosts out of {} using {}", Response.ServerEndPoints.size(), m_Host)); + } + } + else if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(m_OverrideHost, m_AssumeHttp2); TestResult.Success) + { + CloudHost = m_OverrideHost; + } + else + { + throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.FailureReason)); + } + + Oid BuildId = Oid::TryFromHexString(m_BuildId); + if (BuildId == Oid::Zero) + { + throw OptionParseException("'--buildid' is malformed", m_Options.help()); + } + + BuildStorage::Statistics StorageStats; + HttpClient BuildStorageHttp(CloudHost, ClientSettings); + + if (!m_Quiet) + { + std::string StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName), + CloudHost, + BuildStorageHttp.GetSessionId(), + m_Namespace, + m_Bucket); + + ZEN_CONSOLE("Remote: {}", StorageDescription); + } + + std::filesystem::path StorageTempPath = std::filesystem::temp_directory_path() / ("zen_" + Oid::NewOid().ToString()); + + std::unique_ptr<BuildStorage> BuildStorage = + CreateJupiterBuildStorage(Log(), BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, m_AllowRedirect, StorageTempPath); + + Stopwatch Timer; + CbObject BuildObject = BuildStorage->GetBuild(BuildId); + if (!m_Quiet) + { + ZEN_CONSOLE("Fetched {}/{}/{}/{} in {}", m_Url, m_Namespace, m_Bucket, BuildId, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + + Timer.Reset(); + + CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); + if (!PartsObject) + { + throw std::runtime_error( + fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv, m_Url, m_Namespace, m_Bucket, m_BuildId)); + } + + static const std::string_view OplogContainerPartName = "oplogcontainer"sv; + + const Oid OplogBuildPartId = PartsObject[OplogContainerPartName].AsObjectId(); + if (OplogBuildPartId == Oid::Zero) + { + throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv, + m_Url, + m_Namespace, + m_Bucket, + m_BuildId, + OplogContainerPartName)); + } + + CbObject ContainerObject = BuildStorage->GetBuildPart(BuildId, OplogBuildPartId); + + MemoryView OpsSection = ContainerObject["ops"sv].AsBinaryView(); + IoBuffer OpsBuffer(IoBuffer::Wrap, OpsSection.GetData(), OpsSection.GetSize()); + IoBuffer SectionPayload = CompressedBuffer::FromCompressedNoValidate(std::move(OpsBuffer)).Decompress().AsIoBuffer(); + + CbValidateError ValidateResult = CbValidateError::None; + if (CbObject SectionObject = ValidateAndReadCompactBinaryObject(std::move(SectionPayload), ValidateResult); + ValidateResult == CbValidateError::None && ContainerObject) + { + if (!m_Quiet) + { + ZEN_CONSOLE("Decompressed and validated oplog payload {} -> {} in {}", + NiceBytes(OpsSection.GetSize()), + NiceBytes(SectionObject.GetSize()), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + + if (m_OutputPath.empty()) + { + if (!m_Yes) + { + if (OpsSection.GetSize() > 8u * 1024u * 1024u) + { + while (!m_Yes) + { + const std::string Prompt = fmt::format("Do you want to output an oplog of size {} to console? (yes/no) ", + NiceBytes(SectionObject.GetSize())); + printf("%s", Prompt.c_str()); + std::string Reponse; + std::getline(std::cin, Reponse); + Reponse = ToLower(Reponse); + if (Reponse == "y" || Reponse == "yes") + { + m_Yes = true; + } + else if (Reponse == "n" || Reponse == "no") + { + return; + } + } + } + } + ExtendableStringBuilder<1024> SB; + SectionObject.ToJson(SB); + ForEachStrTok(SB.ToView(), '\n', [](std::string_view Row) { + ZEN_CONSOLE("{}", Row); + return true; + }); + } + else + { + Timer.Reset(); + const std::string Extension = ToLower(m_OutputPath.extension().string()); + if (Extension == ".cb" || Extension == ".cbo") + { + WriteFile(m_OutputPath, IoBuffer(IoBuffer::Wrap, SectionObject.GetView().GetData(), SectionObject.GetSize())); + } + else if (Extension == ".json") + { + ExtendableStringBuilder<1024> SB; + SectionObject.ToJson(SB); + WriteFile(m_OutputPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + } + else + { + throw std::runtime_error(fmt::format("Unsupported output extension type '{}'", Extension)); + } + if (!m_Quiet) + { + ZEN_CONSOLE("Wrote {} to '{}' in {}", + NiceBytes(FileSizeFromPath(m_OutputPath)), + m_OutputPath, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + } + } + else + { + throw std::runtime_error( + fmt::format("Failed to parse oplog container: '{}' ('{}')", "Section has unexpected data type", ToString(ValidateResult))); + } +} + } // namespace zen diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index 70b336650..136319aa3 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -4,6 +4,8 @@ #include "../zen.h" +#include "../authutils.h" + namespace zen { class ProjectStoreCommand : public ZenCmdBase @@ -162,6 +164,7 @@ private: std::string m_JupiterNamespace; std::string m_JupiterBucket; + std::string m_JupiterOpenIdProvider; std::string m_JupiterAccessToken; std::string m_JupiterAccessTokenEnv; @@ -268,4 +271,38 @@ private: std::string m_OplogName; }; +class OplogDownloadCommand : public ProjectStoreCommand +{ +public: + static constexpr char Name[] = "oplog-download"; + static constexpr char Description[] = "Download an cloud storage oplog"; + + OplogDownloadCommand(); + ~OplogDownloadCommand(); + virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; + + std::filesystem::path m_SystemRootDir; + + bool m_Quiet = false; + bool m_Yes = false; + + AuthCommandLineOptions m_AuthOptions; + + // cloud builds + std::string m_OverrideHost; + std::string m_Host; + std::string m_Url; + bool m_AssumeHttp2 = false; + bool m_AllowRedirect = false; + std::string m_Namespace; + std::string m_Bucket; + std::string m_BuildId; + + std::filesystem::path m_OutputPath; +}; + } // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 2721433b0..0381dd15c 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -710,6 +710,7 @@ main(int argc, char** argv) JobCommand JobCmd; OplogMirrorCommand OplogMirrorCmd; SnapshotOplogCommand SnapshotOplogCmd; + OplogDownloadCommand OplogDownload; OplogValidateCommand OplogValidateCmd; PrintCommand PrintCmd; PrintPackageCommand PrintPkgCmd; @@ -766,6 +767,7 @@ main(int argc, char** argv) {"oplog-import", &ImportOplogCmd, "Import project store oplog"}, {"oplog-mirror", &OplogMirrorCmd, "Mirror project store oplog to file system"}, {"oplog-snapshot", &SnapshotOplogCmd, "Snapshot project store oplog"}, + {OplogDownloadCommand::Name, &OplogDownload, OplogDownloadCommand::Description}, {"oplog-validate", &OplogValidateCmd, "Validate oplog for missing references"}, {"print", &PrintCmd, "Print compact binary object"}, {"printpackage", &PrintPkgCmd, "Print compact binary package"}, diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp index e5e8db8d2..376d967d1 100644 --- a/src/zenutil/buildstoragecache.cpp +++ b/src/zenutil/buildstoragecache.cpp @@ -413,4 +413,22 @@ CreateZenBuildStorageCache(HttpClient& HttpClient, return std::make_unique<ZenBuildStorageCache>(HttpClient, Stats, Namespace, Bucket, TempFolderPath, BoostBackgroundThreadCount); } +ZenCacheEndpointTestResult +TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2) +{ + HttpClientSettings TestClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); + if (TestResponse.IsSuccess()) + { + return {.Success = true}; + } + return {.Success = false, .FailureReason = TestResponse.ErrorMessage("")}; +}; + } // namespace zen diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h index a0690a16a..e6ca2c5e4 100644 --- a/src/zenutil/include/zenutil/buildstoragecache.h +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -57,4 +57,13 @@ std::unique_ptr<BuildStorageCache> CreateZenBuildStorageCache(HttpClient& H std::string_view Bucket, const std::filesystem::path& TempFolderPath, bool BoostBackgroundThreadCount); + +struct ZenCacheEndpointTestResult +{ + bool Success = false; + std::string FailureReason; +}; + +ZenCacheEndpointTestResult TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2); + } // namespace zen diff --git a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h index bbf070993..f25d8933b 100644 --- a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h +++ b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h @@ -15,4 +15,11 @@ std::unique_ptr<BuildStorage> CreateJupiterBuildStorage(LoggerRef InLog, std::string_view Bucket, bool AllowRedirect, const std::filesystem::path& TempFolderPath); + +bool ParseBuildStorageUrl(std::string_view InUrl, + std::string& OutHost, + std::string& OutNamespace, + std::string& OutBucket, + std::string& OutBuildId); + } // namespace zen diff --git a/src/zenutil/include/zenutil/jupiter/jupiterhost.h b/src/zenutil/include/zenutil/jupiter/jupiterhost.h new file mode 100644 index 000000000..3bbc700b8 --- /dev/null +++ b/src/zenutil/include/zenutil/jupiter/jupiterhost.h @@ -0,0 +1,35 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <string> +#include <string_view> +#include <vector> + +namespace zen { + +struct HttpClientSettings; + +struct JupiterServerDiscovery +{ + struct EndPoint + { + std::string Name; + std::string BaseUrl; + bool AssumeHttp2 = false; + }; + std::vector<EndPoint> ServerEndPoints; + std::vector<EndPoint> CacheEndPoints; +}; + +JupiterServerDiscovery DiscoverJupiterEndpoints(std::string_view Host, const HttpClientSettings& ClientSettings); + +struct JupiterEndpointTestResult +{ + bool Success = false; + std::string FailureReason; +}; + +JupiterEndpointTestResult TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2); + +} // namespace zen diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index 386a91cb3..6eb3489dc 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -14,6 +14,8 @@ ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> ZEN_THIRD_PARTY_INCLUDES_END +#include <regex> + namespace zen { using namespace std::literals; @@ -511,4 +513,49 @@ CreateJupiterBuildStorage(LoggerRef InLog, return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, AllowRedirect, TempFolderPath); } +bool +ParseBuildStorageUrl(std::string_view InUrl, + std::string& OutHost, + std::string& OutNamespace, + std::string& OutBucket, + std::string& OutBuildId) +{ + std::string Url(InUrl); + const std::string_view ExtendedApiString = "api/v2/builds/"; + if (auto ApiString = ToLower(Url).find(ExtendedApiString); ApiString != std::string::npos) + { + Url.erase(ApiString, ExtendedApiString.length()); + } + + const std::string ArtifactURLRegExString = R"((http[s]?:\/\/.*?)\/(.*?)\/(.*?)\/(.*))"; + const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript | std::regex::icase); + std::match_results<std::string_view::const_iterator> MatchResults; + std::string_view UrlToParse(Url); + if (regex_match(begin(UrlToParse), end(UrlToParse), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5) + { + auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view { + ZEN_ASSERT(Index < MatchResults.size()); + + const auto& Match = MatchResults[Index]; + + return std::string_view(&*Match.first, Match.second - Match.first); + }; + + const std::string_view Host = GetMatch(1); + const std::string_view Namespace = GetMatch(2); + const std::string_view Bucket = GetMatch(3); + const std::string_view BuildId = GetMatch(4); + + OutHost = Host; + OutNamespace = Namespace; + OutBucket = Bucket; + OutBuildId = BuildId; + return true; + } + else + { + return false; + } +} + } // namespace zen diff --git a/src/zenutil/jupiter/jupiterhost.cpp b/src/zenutil/jupiter/jupiterhost.cpp new file mode 100644 index 000000000..d06229cbf --- /dev/null +++ b/src/zenutil/jupiter/jupiterhost.cpp @@ -0,0 +1,66 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenutil/jupiter/jupiterhost.h> + +#include <zencore/compactbinary.h> +#include <zencore/fmtutils.h> +#include <zenhttp/httpclient.h> + +namespace zen { + +JupiterServerDiscovery +DiscoverJupiterEndpoints(std::string_view Host, const HttpClientSettings& ClientSettings) +{ + JupiterServerDiscovery Result; + + HttpClient DiscoveryHttpClient(Host, ClientSettings); + HttpClient::Response ServerInfoResponse = DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON)); + if (!ServerInfoResponse.IsSuccess()) + { + ServerInfoResponse.ThrowError(fmt::format("Failed to get list of servers from discovery url '{}'", Host)); + } + std::string_view JsonResponse = ServerInfoResponse.AsText(); + CbObject CbPayload = LoadCompactBinaryFromJson(JsonResponse).AsObject(); + CbArrayView ServerEndpoints = CbPayload["serverEndpoints"].AsArrayView(); + Result.ServerEndPoints.reserve(ServerEndpoints.Num()); + + auto ParseEndPoints = [](CbArrayView ServerEndpoints) -> std::vector<JupiterServerDiscovery::EndPoint> { + std::vector<JupiterServerDiscovery::EndPoint> Result; + + Result.reserve(ServerEndpoints.Num()); + for (CbFieldView ServerEndpointView : ServerEndpoints) + { + CbObjectView ServerEndPoint = ServerEndpointView.AsObjectView(); + Result.push_back(JupiterServerDiscovery::EndPoint{.Name = std::string(ServerEndPoint["name"].AsString()), + .BaseUrl = std::string(ServerEndPoint["baseUrl"].AsString()), + .AssumeHttp2 = ServerEndPoint["baseUrl"].AsBool(false)}); + } + return Result; + }; + + Result.ServerEndPoints = ParseEndPoints(CbPayload["serverEndpoints"].AsArrayView()); + Result.CacheEndPoints = ParseEndPoints(CbPayload["cacheEndpoints"].AsArrayView()); + + return Result; +} + +JupiterEndpointTestResult +TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2) +{ + HttpClientSettings TestClientSettings{.LogCategory = "httpbuildsclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); + if (TestResponse.IsSuccess()) + { + return {.Success = true}; + } + return {.Success = false, .FailureReason = TestResponse.ErrorMessage("")}; +} + +} // namespace zen |