diff options
18 files changed, 628 insertions, 332 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index fd366f26a..b64fa3c7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ - Improvement: Self-hosted dashboard's summary of stats if more robust to future changes - Improvement: Validation of downloaded data in `zen builds download` in now fully opt-in - Improvement: Change the logic for number of network and disk threads for `zen builds` commands +- Improvement: `zen oplog-import` command now does cloud storage host and Zenserver cache discovery + - `--builds-override-host` option can be used instead of `--builds` to specify host to override cloud storage host discovery + - `--zen-cache-host` specifies a Zenserver cache host that overrides any host discovery - Bugfix: Add quotes around messages when using `--log-progress` with `zen builds` commands - Bugfix: Fix ASSERT after running `zen workspace info` on a non-existing workspace. UE-349934 - Bugfix: Fixed issue where GC could start after shutdown of GC had been requested causing a race diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 9b91a9c02..143f567af 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -26,6 +26,7 @@ #include <zenremotestore/builds/buildsavedstate.h> #include <zenremotestore/builds/buildstoragecache.h> #include <zenremotestore/builds/buildstorageoperations.h> +#include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/builds/filebuildstorage.h> #include <zenremotestore/builds/jupiterbuildstorage.h> #include <zenremotestore/chunking/chunkblock.h> @@ -3279,168 +3280,89 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) bool BoostCacheBackgroundWorkerPool) -> StorageInstance { ParseStorageOptions(RequireNamespace, RequireBucket); - StorageInstance Result; + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 2}; + + std::unique_ptr<AuthMgr> Auth; - std::string BuildStorageName = ZEN_CLOUD_STORAGE; - std::string BuildCacheName; - bool CacheAssumeHttp2 = false; std::string StorageDescription; std::string CacheDescription; + StorageInstance Result; + if (!m_Host.empty() || !m_OverrideHost.empty()) { m_AuthOptions.ParseOptions(*SubOption, m_SystemRootDir, ClientSettings, - m_OverrideHost.empty() ? m_Host : m_OverrideHost, + m_Host.empty() ? m_OverrideHost : m_Host, Auth, IsQuiet, /*Hidden*/ false); - } - std::string CloudHost; - - if (!m_Host.empty()) - { - if (m_OverrideHost.empty() || m_ZenCacheHost.empty()) + BuildStorageResolveResult ResolveRes = + ResolveBuildStorage(ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All); + if (!ResolveRes.HostUrl.empty()) { - JupiterServerDiscovery Response = DiscoverJupiterEndpoints(m_Host, ClientSettings); - - if (m_OverrideHost.empty()) + Result.BuildStorageHttp = + std::make_unique<HttpClient>(ResolveRes.HostUrl, ClientSettings, []() { return AbortFlag.load(); }); + + Result.BuildStorage = CreateJupiterBuildStorage(Log(), + *Result.BuildStorageHttp, + StorageStats, + m_Namespace, + m_Bucket, + m_AllowRedirect, + TempPath / "storage"); + Result.StorageName = ResolveRes.HostName; + + StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + ResolveRes.HostName, + (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket); + ; + + if (!ResolveRes.CacheUrl.empty()) { - if (Response.ServerEndPoints.empty()) - { - throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host)); - } - for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : Response.ServerEndPoints) + Result.CacheHttp = std::make_unique<HttpClient>(ResolveRes.CacheUrl, + HttpClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = ResolveRes.CacheAssumeHttp2, + .AllowResume = true, + .RetryCount = 0}, + []() { return AbortFlag.load(); }); + Result.BuildCacheStorage = + CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheName = ResolveRes.CacheName; + + CacheDescription = + fmt::format("Zen {}{}. SessionId: '{}'", + ResolveRes.CacheName, + (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl), + Result.CacheHttp->GetSessionId()); + ; + if (!m_Namespace.empty()) { - if (!ServerEndpoint.BaseUrl.empty()) - { - if (JupiterEndpointTestResult TestResult = - TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2); - TestResult.Success) - { - CloudHost = ServerEndpoint.BaseUrl; - m_AssumeHttp2 = ServerEndpoint.AssumeHttp2; - BuildStorageName = ServerEndpoint.Name; - break; - } - else - { - ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason); - } - } + CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); } - if (CloudHost.empty()) + if (!m_Bucket.empty()) { - throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}", - Response.ServerEndPoints.size(), - m_Host)); + CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); } } - else if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(m_OverrideHost, m_AssumeHttp2); TestResult.Success) - { - CloudHost = m_OverrideHost; - } - else - { - throw std::runtime_error( - fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.FailureReason)); - } - - if (m_ZenCacheHost.empty()) - { - for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : Response.CacheEndPoints) - { - if (!CacheEndpoint.BaseUrl.empty()) - { - if (ZenCacheEndpointTestResult TestResult = - TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2); - TestResult.Success) - { - m_ZenCacheHost = CacheEndpoint.BaseUrl; - CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; - BuildCacheName = CacheEndpoint.Name; - break; - } - } - } - if (m_ZenCacheHost.empty()) - { - ZenServerState State; - if (State.InitializeReadOnly()) - { - State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { - if (m_ZenCacheHost.empty()) - { - std::string ZenServerLocalHostUrl = - fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); - if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenServerLocalHostUrl, false); - TestResult.Success) - { - m_ZenCacheHost = ZenServerLocalHostUrl; - CacheAssumeHttp2 = false; - BuildCacheName = "localhost"; - } - } - }); - } - if (m_ZenCacheHost.empty() && !IsQuiet) - { - ZEN_CONSOLE_WARN("Failed to find any usable cache hosts out of {} using {}", - Response.CacheEndPoints.size(), - m_Host); - } - } - } - else if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, false); TestResult.Success) - { - std::string::size_type HostnameStart = 0; - std::string::size_type HostnameLength = std::string::npos; - if (auto StartPos = m_ZenCacheHost.find("//"); StartPos != std::string::npos) - { - HostnameStart = StartPos + 2; - } - if (auto EndPos = m_ZenCacheHost.find("/", HostnameStart); EndPos != std::string::npos) - { - HostnameLength = EndPos - HostnameStart; - } - BuildCacheName = m_ZenCacheHost.substr(HostnameStart, HostnameLength); - } - else - { - ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.FailureReason); - m_ZenCacheHost = ""; - } - } - else if (!m_OverrideHost.empty()) - { - CloudHost = m_OverrideHost; } } - else - { - CloudHost = m_OverrideHost; - } - - if (!CloudHost.empty()) - { - Result.BuildStorageHttp = std::make_unique<HttpClient>(CloudHost, ClientSettings, []() { return AbortFlag.load(); }); - StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName), - CloudHost, - Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket); - Result.BuildStorage = CreateJupiterBuildStorage(Log(), - *Result.BuildStorageHttp, - StorageStats, - m_Namespace, - m_Bucket, - m_AllowRedirect, - TempPath / "storage"); - Result.StorageName = BuildStorageName; - } else if (!m_StoragePath.empty()) { StorageDescription = fmt::format("folder {}", m_StoragePath); @@ -3451,47 +3373,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help()); } - if (!m_ZenCacheHost.empty()) - { - if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, false); TestResult.Success) - { - Result.CacheHttp = std::make_unique<HttpClient>(m_ZenCacheHost, - HttpClientSettings{.LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{3000}, - .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = CacheAssumeHttp2, - .AllowResume = true, - .RetryCount = 0}, - []() { return AbortFlag.load(); }); - Result.BuildCacheStorage = - CreateZenBuildStorageCache(*Result.CacheHttp, - StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'", - BuildCacheName.empty() ? "" : fmt::format("{}, ", BuildCacheName), - m_ZenCacheHost, - Result.CacheHttp->GetSessionId()); - - if (!m_Namespace.empty()) - { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); - } - if (!m_Bucket.empty()) - { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); - } - Result.CacheName = BuildCacheName.empty() ? m_ZenCacheHost : BuildCacheName; - } - else - { - ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.FailureReason); - m_ZenCacheHost = ""; - } - } + if (!IsQuiet) { ZEN_CONSOLE("Remote: {}", StorageDescription); diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 3f82bf982..e400b96e7 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -19,6 +19,7 @@ #include <zenhttp/httpclient.h> #include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> +#include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/builds/jupiterbuildstorage.h> #include <zenremotestore/jupiter/jupiterhost.h> @@ -1361,9 +1362,20 @@ ImportOplogCommand::ImportOplogCommand() "<assumehttp2>"); m_Options.add_option("", "", "cloud", "Cloud Storage URL", cxxopts::value(m_CloudUrl), "<url>"); + m_Options.add_option("cloud", "", "key", "Cloud Storage key", cxxopts::value(m_CloudKey), "<key>"); - m_Options.add_option("", "", "builds", "Builds Storage URL", cxxopts::value(m_BuildsUrl), "<url>"); + m_Options.add_option("", "", "builds", "Builds Storage URL", cxxopts::value(m_BuildsHost), "<url>"); + + m_Options.add_option("", + "", + "builds-override-host", + "Builds Storage override API host", + cxxopts::value(m_BuildsOverrideHost), + "<override-host>"); + + m_Options + .add_option("builds", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>"); m_Options.add_option("builds", "", "builds-id", "Builds Id", cxxopts::value(m_BuildsId), "<id>"); m_Options.add_option("", "", "zen", "Zen service upload address", cxxopts::value(m_ZenUrl), "<url>"); @@ -1421,7 +1433,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg size_t TargetCount = 0; TargetCount += m_CloudUrl.empty() ? 0 : 1; - TargetCount += m_BuildsUrl.empty() ? 0 : 1; + TargetCount += (m_BuildsHost.empty() && m_BuildsOverrideHost.empty()) ? 0 : 1; TargetCount += m_ZenUrl.empty() ? 0 : 1; TargetCount += m_FileDirectoryPath.empty() ? 0 : 1; if (TargetCount == 0) @@ -1452,7 +1464,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } } - if (!m_BuildsUrl.empty()) + if (!m_BuildsHost.empty() || !m_BuildsOverrideHost.empty()) { if (m_JupiterNamespace.empty()) { @@ -1572,11 +1584,13 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg Writer.EndObject(); // "cloud" SourceDescription = fmt::format("[cloud] {}/{}/{}/{}", m_CloudUrl, m_JupiterNamespace, m_JupiterBucket, m_CloudKey); } - if (!m_BuildsUrl.empty()) + if (!m_BuildsHost.empty() || !m_BuildsOverrideHost.empty()) { Writer.BeginObject("builds"sv); { - Writer.AddString("url"sv, m_BuildsUrl); + Writer.AddString("url"sv, m_BuildsHost); + Writer.AddString("override-host"sv, m_BuildsOverrideHost); + Writer.AddString("zencachehost"sv, m_ZenCacheHost); Writer.AddString("namespace"sv, m_JupiterNamespace); Writer.AddString("bucket"sv, m_JupiterBucket); Writer.AddString("buildsid"sv, m_BuildsId); @@ -2316,27 +2330,13 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a m_Quiet, /*Hidden*/ false); + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ClientSettings, m_Host, m_OverrideHost, ""sv, ZenCacheResolveMode::Off); + +#if 0 std::string BuildStorageName = ZEN_CLOUD_STORAGE; std::string CloudHost; - auto TestHostEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair<bool, std::string> { - HttpClientSettings TestClientSettings{.LogCategory = "httpbuildsclient", - .ConnectTimeout = std::chrono::milliseconds{1000}, - .Timeout = std::chrono::milliseconds{2000}, - .AssumeHttp2 = AssumeHttp2, - .AllowResume = true, - .RetryCount = 0}; - - HttpClient TestHttpClient(BaseUrl, TestClientSettings); - HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); - if (TestResponse.IsSuccess()) - { - return {true, ""}; - } - return {false, TestResponse.ErrorMessage("")}; - }; - if (m_OverrideHost.empty()) { JupiterServerDiscovery Response = DiscoverJupiterEndpoints(m_Host, ClientSettings); @@ -2377,6 +2377,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a { throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.FailureReason)); } +#endif // 0 Oid BuildId = Oid::TryFromHexString(m_BuildId); if (BuildId == Oid::Zero) @@ -2385,16 +2386,16 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } BuildStorage::Statistics StorageStats; - HttpClient BuildStorageHttp(CloudHost, ClientSettings); + HttpClient BuildStorageHttp(ResolveRes.HostUrl, ClientSettings); if (!m_Quiet) { - std::string StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName), - CloudHost, - BuildStorageHttp.GetSessionId(), - m_Namespace, - m_Bucket); + std::string StorageDescription = + fmt::format("Cloud {}{}. Namespace '{}', Bucket '{}'", + ResolveRes.HostName, + (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), + m_Namespace, + m_Bucket); ZEN_CONSOLE("Remote: {}", StorageDescription); } diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index ae8233f51..58002a533 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -172,9 +172,11 @@ private: bool m_JupiterAssumeHttp2 = false; std::string m_CloudUrl; + std::string m_ZenCacheHost; std::string m_CloudKey; - std::string m_BuildsUrl; + std::string m_BuildsHost; + std::string m_BuildsOverrideHost; std::string m_BuildsId; std::string m_ZenUrl; diff --git a/src/zenremotestore/builds/buildstoragecache.cpp b/src/zenremotestore/builds/buildstoragecache.cpp index 2e9cf9954..d7442d7c5 100644 --- a/src/zenremotestore/builds/buildstoragecache.cpp +++ b/src/zenremotestore/builds/buildstoragecache.cpp @@ -359,19 +359,16 @@ public: while (true) { intptr_t Remaining = m_PendingBackgroundWorkCount.Remaining(); - if (UpdateCallback(Remaining)) + if (!UpdateCallback(Remaining)) { - if (m_PendingBackgroundWorkCount.Wait(UpdateIntervalMS)) - { - UpdateCallback(0); - return; - } + m_CancelBackgroundWork.store(true); } - else + if (m_PendingBackgroundWorkCount.Wait(UpdateIntervalMS)) { - m_CancelBackgroundWork.store(true); + break; } } + UpdateCallback(0); } private: diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp new file mode 100644 index 000000000..998529714 --- /dev/null +++ b/src/zenremotestore/builds/buildstorageutil.cpp @@ -0,0 +1,157 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenremotestore/builds/buildstorageutil.h> + +#include <zenremotestore/builds/buildstoragecache.h> +#include <zenremotestore/builds/jupiterbuildstorage.h> +#include <zenremotestore/jupiter/jupiterhost.h> +#include <zenutil/zenserverprocess.h> + +namespace zen { + +BuildStorageResolveResult +ResolveBuildStorage(const HttpClientSettings& ClientSettings, + std::string_view Host, + std::string_view OverrideHost, + std::string_view ZenCacheHost, + ZenCacheResolveMode ZenResolveMode) +{ + bool AllowZenCacheDiscovery = ZenResolveMode == ZenCacheResolveMode::Discovery || ZenResolveMode == ZenCacheResolveMode::All; + bool AllowLocalZenCache = ZenResolveMode == ZenCacheResolveMode::LocalHost || ZenResolveMode == ZenCacheResolveMode::All; + + auto GetHostNameFromUrl = [](std::string_view Url) -> std::string_view { + std::string::size_type HostnameStart = 0; + std::string::size_type HostnameLength = std::string::npos; + if (auto StartPos = Url.find("//"); StartPos != std::string::npos) + { + HostnameStart = StartPos + 2; + } + if (auto EndPos = Url.find("/", HostnameStart); EndPos != std::string::npos) + { + HostnameLength = EndPos - HostnameStart; + } + if (auto EndPos = Url.find(":", HostnameStart); EndPos != std::string::npos) + { + HostnameLength = EndPos - HostnameStart; + } + return Url.substr(HostnameStart, HostnameLength); + }; + + std::string HostUrl; + std::string HostName; + + std::string CacheUrl; + std::string CacheName; + bool HostAssumeHttp2 = ClientSettings.AssumeHttp2; + bool CacheAssumeHttp2 = ClientSettings.AssumeHttp2; + + JupiterServerDiscovery DiscoveryResponse; + const std::string_view DiscoveryHost = Host.empty() ? OverrideHost : Host; + + if (OverrideHost.empty() || (ZenCacheHost.empty() && AllowZenCacheDiscovery)) + { + DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings); + } + + if (!OverrideHost.empty()) + { + if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2); TestResult.Success) + { + HostUrl = OverrideHost; + HostName = GetHostNameFromUrl(OverrideHost); + } + else + { + throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", OverrideHost, TestResult.FailureReason)); + } + } + else + { + if (DiscoveryResponse.ServerEndPoints.empty()) + { + throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", DiscoveryHost)); + } + + for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : DiscoveryResponse.ServerEndPoints) + { + if (!ServerEndpoint.BaseUrl.empty()) + { + if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2); + TestResult.Success) + { + HostUrl = ServerEndpoint.BaseUrl; + HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; + HostName = ServerEndpoint.Name; + break; + } + else + { + ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason); + } + } + } + if (HostUrl.empty()) + { + throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}", + DiscoveryResponse.ServerEndPoints.size(), + DiscoveryHost)); + } + } + if (ZenCacheHost.empty()) + { + if (AllowZenCacheDiscovery) + { + for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : DiscoveryResponse.CacheEndPoints) + { + if (!CacheEndpoint.BaseUrl.empty()) + { + if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2); + TestResult.Success) + { + CacheUrl = CacheEndpoint.BaseUrl; + CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; + CacheName = CacheEndpoint.Name; + break; + } + } + } + } + if (CacheUrl.empty() && AllowLocalZenCache) + { + ZenServerState State; + if (State.InitializeReadOnly()) + { + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + if (CacheUrl.empty()) + { + std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); + if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenServerLocalHostUrl, false); TestResult.Success) + { + CacheUrl = ZenServerLocalHostUrl; + CacheAssumeHttp2 = false; + CacheName = "localhost"; + } + } + }); + } + } + } + else if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, false); TestResult.Success) + { + CacheUrl = ZenCacheHost; + CacheName = GetHostNameFromUrl(ZenCacheHost); + } + else + { + ZEN_WARN("Unable to reach cache host {}. Reason: {}", ZenCacheHost, TestResult.FailureReason); + } + + return BuildStorageResolveResult{.HostUrl = HostUrl, + .HostName = HostName, + .HostAssumeHttp2 = HostAssumeHttp2, + + .CacheUrl = CacheUrl, + .CacheName = CacheName, + .CacheAssumeHttp2 = CacheAssumeHttp2}; +} +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h new file mode 100644 index 000000000..258266a6a --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h @@ -0,0 +1,40 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging.h> +#include <zenhttp/httpclient.h> +#include <zenremotestore/builds/buildstorage.h> + +namespace cxxopts { +class Options; +} + +namespace zen { + +struct BuildStorageResolveResult +{ + std::string HostUrl; + std::string HostName; + bool HostAssumeHttp2 = false; + + std::string CacheUrl; + std::string CacheName; + bool CacheAssumeHttp2 = false; +}; + +enum class ZenCacheResolveMode +{ + Off, + Discovery, + LocalHost, + All +}; + +BuildStorageResolveResult ResolveBuildStorage(const HttpClientSettings& ClientSettings, + std::string_view Host, + std::string_view OverrideHost, + std::string_view ZenCacheHost, + ZenCacheResolveMode ZenResolveMode); + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h index 037325ed1..ecd157af8 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h @@ -10,7 +10,9 @@ class AuthMgr; struct BuildsRemoteStoreOptions : RemoteStoreOptions { - std::string Url; + std::string Host; + std::string OverrideHost; + std::string ZenHost; std::string Namespace; std::string Bucket; Oid BuildId; @@ -24,10 +26,12 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions IoBuffer MetaData; }; -std::shared_ptr<RemoteProjectStore> CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, +std::shared_ptr<RemoteProjectStore> CreateJupiterBuildsRemoteStore(LoggerRef InLog, + const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet, bool Unattended, - bool Hidden); + bool Hidden, + WorkerThreadPool& CacheBackgroundWorkerPool); } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h index ff2ecb405..72544363f 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h @@ -15,6 +15,6 @@ struct FileRemoteStoreOptions : RemoteStoreOptions bool ForceEnableTempBlocks = false; }; -std::shared_ptr<RemoteProjectStore> CreateFileRemoteStore(const FileRemoteStoreOptions& Options); +std::shared_ptr<RemoteProjectStore> CreateFileRemoteStore(LoggerRef InLog, const FileRemoteStoreOptions& Options); } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h index 13a346039..aeeb5c0d0 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h @@ -24,7 +24,8 @@ struct JupiterRemoteStoreOptions : RemoteStoreOptions bool AssumeHttp2 = false; }; -std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, +std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(LoggerRef InLog, + const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet, bool Unattended, diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h index fbcdde955..4740e7029 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h @@ -105,6 +105,8 @@ public: virtual GetKnownBlocksResult GetKnownBlocks() = 0; virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) = 0; virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0; + + virtual void Flush() = 0; }; struct RemoteStoreOptions diff --git a/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h index d6b62c563..2188be838 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h @@ -13,6 +13,8 @@ struct ZenRemoteStoreOptions : RemoteStoreOptions std::string OplogId; }; -std::shared_ptr<RemoteProjectStore> CreateZenRemoteStore(const ZenRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath); +std::shared_ptr<RemoteProjectStore> CreateZenRemoteStore(LoggerRef InLog, + const ZenRemoteStoreOptions& Options, + const std::filesystem::path& TempFilePath); } // namespace zen diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp index 2cc8ed4aa..1d749a17d 100644 --- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp @@ -8,8 +8,12 @@ #include <zencore/scopeguard.h> #include <zenhttp/httpclientauth.h> +#include <zenremotestore/builds/buildstoragecache.h> +#include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/builds/jupiterbuildstorage.h> +#include <numeric> + namespace zen { using namespace std::literals; @@ -19,20 +23,28 @@ static const std::string_view OplogContainerPartName = "oplogcontainer"sv; class BuildsRemoteStore : public RemoteProjectStore { public: - BuildsRemoteStore(std::unique_ptr<BuildStorage::Statistics>&& BuildStorageStats, - std::unique_ptr<HttpClient>&& BuildStorageHttp, - std::unique_ptr<BuildStorage>&& BuildStorage, - std::string_view Url, - std::string_view Namespace, - std::string_view Bucket, - const Oid& BuildId, - const IoBuffer& MetaData, - bool ForceDisableBlocks, - bool ForceDisableTempBlocks) - : m_BuildStorageStats(std::move(BuildStorageStats)) - , m_BuildStorageHttp(std::move(BuildStorageHttp)) - , m_BuildStorage(std::move(BuildStorage)) - , m_Url(Url) + BuildsRemoteStore(LoggerRef InLog, + const HttpClientSettings& ClientSettings, + HttpClientSettings* OptionalCacheClientSettings, + std::string_view HostUrl, + std::string_view CacheUrl, + const std::filesystem::path& TempFilePath, + WorkerThreadPool& CacheBackgroundWorkerPool, + std::string_view Namespace, + std::string_view Bucket, + const Oid& BuildId, + const IoBuffer& MetaData, + bool ForceDisableBlocks, + bool ForceDisableTempBlocks) + : m_Log(InLog) + , m_BuildStorageHttp(HostUrl, ClientSettings) + , m_BuildStorage(CreateJupiterBuildStorage(Log(), + m_BuildStorageHttp, + m_BuildStorageStats, + Namespace, + Bucket, + /*AllowRedirect*/ false, + TempFilePath)) , m_Namespace(Namespace) , m_Bucket(Bucket) , m_BuildId(BuildId) @@ -41,6 +53,17 @@ public: , m_UseTempBlocks(!ForceDisableTempBlocks) { m_MetaData.MakeOwned(); + if (OptionalCacheClientSettings) + { + ZEN_ASSERT(!CacheUrl.empty()); + m_BuildCacheStorageHttp = std::make_unique<HttpClient>(CacheUrl, *OptionalCacheClientSettings); + m_BuildCacheStorage = CreateZenBuildStorageCache(*m_BuildCacheStorageHttp, + m_StorageCacheStats, + Namespace, + Bucket, + TempFilePath, + CacheBackgroundWorkerPool); + } } virtual RemoteStoreInfo GetInfo() const override @@ -49,9 +72,10 @@ public: .UseTempBlockFiles = m_UseTempBlocks, .AllowChunking = true, .ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId), - .Description = fmt::format("[cloud] {}. SessionId: {}. {}/{}/{}"sv, - m_Url, - m_BuildStorageHttp->GetSessionId(), + .Description = fmt::format("[cloud] {}{}. SessionId: {}. {}/{}/{}"sv, + m_BuildStorageHttp.GetBaseUri(), + m_BuildCacheStorage ? fmt::format(" (Cache: {})", m_BuildCacheStorageHttp->GetBaseUri()) : ""sv, + m_BuildStorageHttp.GetSessionId(), m_Namespace, m_Bucket, m_BuildId)}; @@ -60,13 +84,13 @@ public: virtual Stats GetStats() const override { return { - .m_SentBytes = m_BuildStorageStats->TotalBytesWritten.load(), - .m_ReceivedBytes = m_BuildStorageStats->TotalBytesRead.load(), - .m_RequestTimeNS = m_BuildStorageStats->TotalRequestTimeUs.load() * 1000, - .m_RequestCount = m_BuildStorageStats->TotalRequestCount.load(), - .m_PeakSentBytes = m_BuildStorageStats->PeakSentBytes.load(), - .m_PeakReceivedBytes = m_BuildStorageStats->PeakReceivedBytes.load(), - .m_PeakBytesPerSec = m_BuildStorageStats->PeakBytesPerSec.load(), + .m_SentBytes = m_BuildStorageStats.TotalBytesWritten.load() + m_StorageCacheStats.TotalBytesWritten.load(), + .m_ReceivedBytes = m_BuildStorageStats.TotalBytesRead.load() + m_StorageCacheStats.TotalBytesRead.load(), + .m_RequestTimeNS = m_BuildStorageStats.TotalRequestTimeUs.load() * 1000 + m_StorageCacheStats.TotalRequestTimeUs.load() * 1000, + .m_RequestCount = m_BuildStorageStats.TotalRequestCount.load() + m_StorageCacheStats.TotalRequestCount.load(), + .m_PeakSentBytes = Max(m_BuildStorageStats.PeakSentBytes.load(), m_StorageCacheStats.PeakSentBytes.load()), + .m_PeakReceivedBytes = Max(m_BuildStorageStats.PeakReceivedBytes.load(), m_StorageCacheStats.PeakReceivedBytes.load()), + .m_PeakBytesPerSec = Max(m_BuildStorageStats.PeakBytesPerSec.load(), m_StorageCacheStats.PeakBytesPerSec.load()), }; } @@ -88,14 +112,22 @@ public: catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); - Result.Reason = - fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); + Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = - fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); + Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); } return Result; @@ -122,7 +154,7 @@ public: { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -133,7 +165,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -168,7 +200,7 @@ public: if (!m_BuildStorage->PutBlockMetadata(m_BuildId, RawHash, MetaPayload)) { ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -180,7 +212,7 @@ public: { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -190,7 +222,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -202,7 +234,7 @@ public: { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -212,7 +244,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -259,7 +291,7 @@ public: : Ex.GetHttpResponseCode() != HttpResponseCode::ImATeapot ? (int)Ex.GetHttpResponseCode() : 0; Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -270,7 +302,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -290,7 +322,7 @@ public: : Ex.GetHttpResponseCode() != HttpResponseCode::ImATeapot ? (int)Ex.GetHttpResponseCode() : 0; Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -300,7 +332,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -327,7 +359,7 @@ public: if (!PartsObject) { throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv, - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId)); @@ -336,7 +368,7 @@ public: if (m_OplogBuildPartId == Oid::Zero) { throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv, - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -349,7 +381,7 @@ public: { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -359,7 +391,7 @@ public: { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'", - m_Url, + m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, @@ -391,14 +423,22 @@ public: catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); - Result.Reason = - fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); + Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = - fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); + Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); } return Result; @@ -414,19 +454,45 @@ public: try { - Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash); + if (m_BuildCacheStorage) + { + IoBuffer CachedBlob = m_BuildCacheStorage->GetBuildBlob(m_BuildId, RawHash); + if (CachedBlob) + { + Result.Bytes = std::move(CachedBlob); + } + } + if (!Result.Bytes) + { + Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash); + if (m_BuildCacheStorage && Result.Bytes) + { + m_BuildCacheStorage->PutBuildBlob(m_BuildId, + RawHash, + Result.Bytes.GetContentType(), + CompositeBuffer(SharedBuffer(Result.Bytes))); + } + } } catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); - Result.Reason = - fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); + Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = - fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); + Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); } return Result; @@ -437,7 +503,42 @@ public: LoadAttachmentsResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; }); - for (const IoHash& Hash : RawHashes) + + std::vector<IoHash> AttachmentsLeftToFind = RawHashes; + + if (m_BuildCacheStorage) + { + std::vector<BuildStorageCache::BlobExistsResult> ExistCheck = m_BuildCacheStorage->BlobsExists(m_BuildId, RawHashes); + if (ExistCheck.size() == RawHashes.size()) + { + AttachmentsLeftToFind.clear(); + for (size_t BlobIndex = 0; BlobIndex < RawHashes.size(); BlobIndex++) + { + const IoHash& Hash = RawHashes[BlobIndex]; + const BuildStorageCache::BlobExistsResult& BlobExists = ExistCheck[BlobIndex]; + if (BlobExists.HasBody) + { + IoBuffer CachedPayload = m_BuildCacheStorage->GetBuildBlob(m_BuildId, Hash); + if (CachedPayload) + { + Result.Chunks.emplace_back( + std::pair<IoHash, CompressedBuffer>{Hash, + CompressedBuffer::FromCompressedNoValidate(std::move(CachedPayload))}); + } + else + { + AttachmentsLeftToFind.push_back(Hash); + } + } + else + { + AttachmentsLeftToFind.push_back(Hash); + } + } + } + } + + for (const IoHash& Hash : AttachmentsLeftToFind) { LoadAttachmentResult ChunkResult = LoadAttachment(Hash); if (ChunkResult.ErrorCode) @@ -445,12 +546,27 @@ public: return LoadAttachmentsResult{ChunkResult}; } ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(ChunkResult.ElapsedSeconds * 1000))); + if (m_BuildCacheStorage && ChunkResult.Bytes) + { + m_BuildCacheStorage->PutBuildBlob(m_BuildId, + Hash, + ChunkResult.Bytes.GetContentType(), + CompositeBuffer(SharedBuffer(ChunkResult.Bytes))); + } Result.Chunks.emplace_back( std::pair<IoHash, CompressedBuffer>{Hash, CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult.Bytes))}); } return Result; } + virtual void Flush() override + { + if (m_BuildCacheStorage) + { + m_BuildCacheStorage->Flush(100, [](intptr_t) { return false; }); + } + } + private: static int MakeErrorCode(const HttpClientError& Ex) { @@ -459,32 +575,54 @@ private: : 0; } - std::unique_ptr<BuildStorage::Statistics> m_BuildStorageStats; - std::unique_ptr<HttpClient> m_BuildStorageHttp; - std::unique_ptr<BuildStorage> m_BuildStorage; - const std::string m_Url; - const std::string m_Namespace; - const std::string m_Bucket; - const Oid m_BuildId; - IoBuffer m_MetaData; - Oid m_OplogBuildPartId = Oid::Zero; - const bool m_EnableBlocks = true; - const bool m_UseTempBlocks = true; - const bool m_AllowRedirect = false; + inline LoggerRef Log() const { return m_Log; } + + LoggerRef m_Log; + + BuildStorage::Statistics m_BuildStorageStats; + HttpClient m_BuildStorageHttp; + std::unique_ptr<BuildStorage> m_BuildStorage; + + BuildStorageCache::Statistics m_StorageCacheStats; + std::unique_ptr<HttpClient> m_BuildCacheStorageHttp; + std::unique_ptr<BuildStorageCache> m_BuildCacheStorage; + + const std::string m_Namespace; + const std::string m_Bucket; + const Oid m_BuildId; + IoBuffer m_MetaData; + Oid m_OplogBuildPartId = Oid::Zero; + const bool m_EnableBlocks = true; + const bool m_UseTempBlocks = true; + const bool m_AllowRedirect = false; }; std::shared_ptr<RemoteProjectStore> -CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, +CreateJupiterBuildsRemoteStore(LoggerRef InLog, + const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet, bool Unattended, - bool Hidden) + bool Hidden, + WorkerThreadPool& CacheBackgroundWorkerPool) { - std::string Url = Options.Url; - if (Url.find("://"sv) == std::string::npos) + std::string Host = Options.Host; + if (!Host.empty() && Host.find("://"sv) == std::string::npos) + { + // Assume https URL + Host = fmt::format("https://{}"sv, Host); + } + std::string OverrideUrl = Options.OverrideHost; + if (!OverrideUrl.empty() && OverrideUrl.find("://"sv) == std::string::npos) + { + // Assume https URL + OverrideUrl = fmt::format("https://{}"sv, OverrideUrl); + } + std::string ZenHost = Options.ZenHost; + if (!ZenHost.empty() && ZenHost.find("://"sv) == std::string::npos) { // Assume https URL - Url = fmt::format("https://{}"sv, Url); + ZenHost = fmt::format("https://{}"sv, ZenHost); } // 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider @@ -503,7 +641,11 @@ CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, } else if (!Options.OidcExePath.empty()) { - if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet, Unattended, Hidden); + if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, + Host.empty() ? OverrideUrl : Host, + Quiet, + Unattended, + Hidden); TokenProviderMaybe) { TokenProvider = TokenProviderMaybe.value(); @@ -515,25 +657,44 @@ CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager); } + BuildStorageResolveResult ResolveRes; + { + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AccessTokenProvider = TokenProvider, + .AssumeHttp2 = Options.AssumeHttp2, + .AllowResume = true, + .RetryCount = 2}; + + ResolveRes = ResolveBuildStorage(ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery); + } + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", .ConnectTimeout = std::chrono::milliseconds(2000), .Timeout = std::chrono::milliseconds(1800000), .AccessTokenProvider = std::move(TokenProvider), - .AssumeHttp2 = Options.AssumeHttp2, + .AssumeHttp2 = ResolveRes.HostAssumeHttp2, .AllowResume = true, .RetryCount = 4}; - std::unique_ptr<BuildStorage::Statistics> BuildStorageStats(std::make_unique<BuildStorage::Statistics>()); - - std::unique_ptr<HttpClient> BuildStorageHttp = std::make_unique<HttpClient>(Url, ClientSettings); + std::unique_ptr<HttpClientSettings> CacheClientSettings; - std::unique_ptr<BuildStorage> BuildStorage = - CreateJupiterBuildStorage(Log(), *BuildStorageHttp, *BuildStorageStats, Options.Namespace, Options.Bucket, false, TempFilePath); + if (!ResolveRes.CacheUrl.empty()) + { + CacheClientSettings = std::make_unique<HttpClientSettings>(HttpClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = ResolveRes.CacheAssumeHttp2, + .AllowResume = true, + .RetryCount = 0}); + } - std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(BuildStorageStats), - std::move(BuildStorageHttp), - std::move(BuildStorage), - Url, + std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(InLog, + ClientSettings, + CacheClientSettings.get(), + ResolveRes.HostUrl, + ResolveRes.CacheUrl, + TempFilePath, + CacheBackgroundWorkerPool, Options.Namespace, Options.Bucket, Options.BuildId, diff --git a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp index d6e6944f4..f49f0d72e 100644 --- a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp @@ -17,12 +17,14 @@ using namespace std::literals; class LocalExportProjectStore : public RemoteProjectStore { public: - LocalExportProjectStore(std::string_view Name, + LocalExportProjectStore(LoggerRef InLog, + std::string_view Name, std::string_view OptionalBaseName, const std::filesystem::path& FolderPath, bool ForceDisableBlocks, bool ForceEnableTempBlocks) - : m_Name(Name) + : m_Log(InLog) + , m_Name(Name) , m_OptionalBaseName(OptionalBaseName) , m_OutputPath(FolderPath) { @@ -241,6 +243,8 @@ public: return Result; } + virtual void Flush() override {} + private: LoadContainerResult LoadContainer(const std::string& Name) { @@ -312,6 +316,10 @@ private: m_RequestCount.fetch_add(1); } + inline LoggerRef Log() const { return m_Log; } + + LoggerRef m_Log; + const std::string m_Name; const std::string m_OptionalBaseName; const std::filesystem::path m_OutputPath; @@ -328,9 +336,10 @@ private: }; std::shared_ptr<RemoteProjectStore> -CreateFileRemoteStore(const FileRemoteStoreOptions& Options) +CreateFileRemoteStore(LoggerRef InLog, const FileRemoteStoreOptions& Options) { - std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<LocalExportProjectStore>(Options.Name, + std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<LocalExportProjectStore>(InLog, + Options.Name, Options.OptionalBaseName, std::filesystem::path(Options.FolderPath), Options.ForceDisableBlocks, diff --git a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp index dda5ef99d..ca7a5b391 100644 --- a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp @@ -18,7 +18,8 @@ using namespace std::literals; class JupiterRemoteStore : public RemoteProjectStore { public: - JupiterRemoteStore(Ref<JupiterClient>&& InJupiterClient, + JupiterRemoteStore(LoggerRef InLog, + Ref<JupiterClient>&& InJupiterClient, std::string_view Namespace, std::string_view Bucket, const IoHash& Key, @@ -26,7 +27,8 @@ public: bool ForceDisableBlocks, bool ForceDisableTempBlocks, const std::filesystem::path& TempFilePath) - : m_JupiterClient(std::move(InJupiterClient)) + : m_Log(InLog) + , m_JupiterClient(std::move(InJupiterClient)) , m_Namespace(Namespace) , m_Bucket(Bucket) , m_Key(Key) @@ -232,6 +234,8 @@ public: return Result; } + virtual void Flush() override {} + private: LoadContainerResult LoadContainer(const IoHash& Key) { @@ -323,6 +327,10 @@ private: return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text}; } + inline LoggerRef Log() const { return m_Log; } + + LoggerRef m_Log; + Ref<JupiterClient> m_JupiterClient; const std::string m_Namespace; const std::string m_Bucket; @@ -343,7 +351,8 @@ private: }; std::shared_ptr<RemoteProjectStore> -CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, +CreateJupiterRemoteStore(LoggerRef InLog, + const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet, bool Unattended, @@ -392,7 +401,8 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider))); - std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<JupiterRemoteStore>(std::move(Client), + std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<JupiterRemoteStore>(InLog, + std::move(Client), Options.Namespace, Options.Bucket, Options.Key, diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index 727a25ba1..d316f67e5 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -3276,6 +3276,7 @@ LoadOplog(CidStore& ChunkStore, { if (CleanOplog) { + RemoteStore.Flush(); if (!Oplog.Reset()) { Result = RemoteProjectStore::Result{.ErrorCode = gsl::narrow<int>(HttpResponseCode::InternalServerError), @@ -3449,7 +3450,7 @@ TEST_CASE_TEMPLATE("project.store.export", /*OptionalBaseName = */ std::string(), /*.ForceDisableBlocks = */ Settings::ForceDisableBlocks, /*.ForceEnableTempBlocks = */ Settings::ForceEnableTempBlocks}; - std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options); + std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Log(), Options); RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); uint32_t NetworkWorkerCount = Max(GetHardwareConcurrency() / 4u, 2u); diff --git a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp index 000901e45..e520466e0 100644 --- a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp @@ -17,11 +17,13 @@ using namespace std::literals; class ZenRemoteStore : public RemoteProjectStore { public: - ZenRemoteStore(std::string_view HostAddress, + ZenRemoteStore(LoggerRef InLog, + std::string_view HostAddress, std::string_view Project, std::string_view Oplog, const std::filesystem::path& TempFilePath) - : m_HostAddress(HostAddress) + : m_Log(InLog) + , m_HostAddress(HostAddress) , m_ProjectStoreUrl(fmt::format("{}/prj"sv, m_HostAddress)) , m_Project(Project) , m_Oplog(Oplog) @@ -266,6 +268,8 @@ public: return Result; } + virtual void Flush() override {} + private: void AddStats(const HttpClient::Response& Result) { @@ -302,6 +306,10 @@ private: return {.ErrorCode = 0, .ElapsedSeconds = Response.ElapsedSeconds}; } + inline LoggerRef Log() const { return m_Log; } + + LoggerRef m_Log; + const std::string m_HostAddress; const std::string m_ProjectStoreUrl; const std::string m_Project; @@ -320,7 +328,7 @@ private: }; std::shared_ptr<RemoteProjectStore> -CreateZenRemoteStore(const ZenRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath) +CreateZenRemoteStore(LoggerRef InLog, const ZenRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath) { std::string Url = Options.Url; if (Url.find("://"sv) == std::string::npos) @@ -329,7 +337,7 @@ CreateZenRemoteStore(const ZenRemoteStoreOptions& Options, const std::filesystem Url = fmt::format("http://{}"sv, Url); } std::shared_ptr<RemoteProjectStore> RemoteStore = - std::make_shared<ZenRemoteStore>(Url, Options.ProjectId, Options.OplogId, TempFilePath); + std::make_shared<ZenRemoteStore>(InLog, Url, Options.ProjectId, Options.OplogId, TempFilePath); return RemoteStore; } diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp index 1c6b5d6b0..eef9fa7e6 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.cpp +++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp @@ -245,7 +245,8 @@ namespace { std::string Description; }; - CreateRemoteStoreResult CreateRemoteStore(CbObjectView Params, + CreateRemoteStoreResult CreateRemoteStore(LoggerRef InLog, + CbObjectView Params, AuthMgr& AuthManager, size_t MaxBlockSize, size_t MaxChunkEmbedSize, @@ -253,6 +254,8 @@ namespace { { ZEN_MEMSCOPE(GetProjectHttpTag()); + auto Log = [InLog]() { return InLog; }; + using namespace std::literals; std::shared_ptr<RemoteProjectStore> RemoteStore; @@ -280,7 +283,7 @@ namespace { std::string(OptionalBaseName), ForceDisableBlocks, ForceEnableTempBlocks}; - RemoteStore = CreateFileRemoteStore(Options); + RemoteStore = CreateFileRemoteStore(Log(), Options); } if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud) @@ -372,7 +375,7 @@ namespace { ForceDisableBlocks, ForceDisableTempBlocks, AssumeHttp2}; - RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + RemoteStore = CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); } if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) @@ -393,19 +396,24 @@ namespace { std::string(Url), std::string(Project), std::string(Oplog)}; - RemoteStore = CreateZenRemoteStore(Options, TempFilePath); + RemoteStore = CreateZenRemoteStore(Log(), Options, TempFilePath); } if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds) { - std::string_view BuildsServiceUrl = Builds["url"sv].AsString(); - if (BuildsServiceUrl.empty()) + std::string_view BuildsServiceHost = Builds["url"sv].AsString(); + std::string_view BuildsServiceOverrideHost = Builds["override-host"sv].AsString(); + if (BuildsServiceHost.empty() && BuildsServiceOverrideHost.empty()) { - return {nullptr, "Missing service url"}; + return {nullptr, "Missing service url or host"}; } - std::string Url = UrlDecode(BuildsServiceUrl); - std::string_view Namespace = Builds["namespace"sv].AsString(); + std::string_view BuildsZenCacheHost = Builds["zencachehost"sv].AsString(); + + std::string Host = UrlDecode(BuildsServiceHost); + std::string OverrideHost = UrlDecode(BuildsServiceOverrideHost); + std::string ZenHost = UrlDecode(BuildsZenCacheHost); + std::string_view Namespace = Builds["namespace"sv].AsString(); if (Namespace.empty()) { return {nullptr, "Missing namespace"}; @@ -462,7 +470,9 @@ namespace { BuildsRemoteStoreOptions Options = { RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - Url, + Host, + OverrideHost, + ZenHost, std::string(Namespace), std::string(Bucket), BuildId, @@ -474,7 +484,13 @@ namespace { ForceDisableTempBlocks, AssumeHttp2, MetaData}; - RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + RemoteStore = CreateJupiterBuildsRemoteStore(Log(), + Options, + TempFilePath, + /*Quiet*/ false, + /*Unattended*/ false, + /*Hidden*/ true, + GetTinyWorkerPool(EWorkloadType::Background)); } if (!RemoteStore) @@ -2623,7 +2639,7 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) bool CleanOplog = Params["clean"].AsBool(false); CreateRemoteStoreResult RemoteStoreResult = - CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); + CreateRemoteStore(Log(), Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); if (RemoteStoreResult.Store == nullptr) { @@ -2681,7 +2697,7 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) bool EmbedLooseFile = Params["embedloosefiles"sv].AsBool(false); CreateRemoteStoreResult RemoteStoreResult = - CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); + CreateRemoteStore(Log(), Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); if (RemoteStoreResult.Store == nullptr) { |