diff options
Diffstat (limited to 'src/zenremotestore/builds/buildstorageresolve.cpp')
| -rw-r--r-- | src/zenremotestore/builds/buildstorageresolve.cpp | 249 |
1 files changed, 249 insertions, 0 deletions
diff --git a/src/zenremotestore/builds/buildstorageresolve.cpp b/src/zenremotestore/builds/buildstorageresolve.cpp new file mode 100644 index 000000000..b33d7af29 --- /dev/null +++ b/src/zenremotestore/builds/buildstorageresolve.cpp @@ -0,0 +1,249 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenremotestore/builds/buildstorageresolve.h> + +#include <zencore/fmtutils.h> +#include <zenremotestore/builds/buildstoragecache.h> +#include <zenremotestore/jupiter/jupiterhost.h> +#include <zenutil/zenserverprocess.h> + +namespace zen { + +namespace { + std::string ConnectionSettingsToString(const HttpClientSettings& ClientSettings) + { + ExtendableStringBuilder<128> SB; + SB << "\n LogCategory: " << ClientSettings.LogCategory; + SB << "\n ConnectTimeout: " << ClientSettings.ConnectTimeout.count() << " ms"; + SB << "\n Timeout: " << ClientSettings.Timeout.count() << " ms"; + SB << "\n AccessTokenProvider: " << ClientSettings.AccessTokenProvider.has_value(); + SB << "\n AssumeHttp2: " << ClientSettings.AssumeHttp2; + SB << "\n AllowResume: " << ClientSettings.AllowResume; + SB << "\n RetryCount: " << ClientSettings.RetryCount; + SB << "\n SessionId: " << ClientSettings.SessionId.ToString(); + SB << "\n Verbose: " << ClientSettings.Verbose; + SB << "\n MaximumInMemoryDownloadSize: " << ClientSettings.MaximumInMemoryDownloadSize; + return SB.ToString(); + } +} // namespace + +BuildStorageResolveResult +ResolveBuildStorage(LoggerRef InLog, + const HttpClientSettings& ClientSettings, + std::string_view Host, + std::string_view OverrideHost, + std::string_view ZenCacheHost, + ZenCacheResolveMode ZenResolveMode, + bool Verbose) +{ + ZEN_SCOPED_LOG(InLog); + + bool AllowZenCacheDiscovery = ZenResolveMode == ZenCacheResolveMode::Discovery || ZenResolveMode == ZenCacheResolveMode::All; + bool AllowLocalZenCache = ZenResolveMode == ZenCacheResolveMode::LocalHost || ZenResolveMode == ZenCacheResolveMode::All; + + auto GetHostNameFromUrl = [](std::string_view Url) -> std::string_view { + std::string::size_type HostnameStart = 0; + std::string::size_type HostnameLength = std::string::npos; + if (auto StartPos = Url.find("//"); StartPos != std::string::npos) + { + HostnameStart = StartPos + 2; + } + if (auto EndPos = Url.find("/", HostnameStart); EndPos != std::string::npos) + { + HostnameLength = EndPos - HostnameStart; + } + if (auto EndPos = Url.find(":", HostnameStart); EndPos != std::string::npos) + { + HostnameLength = EndPos - HostnameStart; + } + return Url.substr(HostnameStart, HostnameLength); + }; + + std::string HostUrl; + std::string HostName; + double HostLatencySec = -1.0; + uint64_t HostMaxRangeCountPerRequest = 1; + + std::string CacheUrl; + std::string CacheName; + bool HostAssumeHttp2 = ClientSettings.AssumeHttp2; + bool CacheAssumeHttp2 = ClientSettings.AssumeHttp2; + double CacheLatencySec = -1.0; + uint64_t CacheMaxRangeCountPerRequest = 1; + + JupiterServerDiscovery DiscoveryResponse; + const std::string_view DiscoveryHost = Host.empty() ? OverrideHost : Host; + + if (OverrideHost.empty() || (ZenCacheHost.empty() && AllowZenCacheDiscovery)) + { + if (Verbose) + { + ZEN_INFO("Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}", + DiscoveryHost, + ConnectionSettingsToString(ClientSettings)); + } + + DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings); + } + + if (!OverrideHost.empty()) + { + if (Verbose) + { + ZEN_INFO("Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2); + } + if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2, ClientSettings.Verbose); + TestResult.Success) + { + if (Verbose) + { + ZEN_INFO("Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost); + } + HostUrl = OverrideHost; + HostName = GetHostNameFromUrl(OverrideHost); + HostLatencySec = TestResult.LatencySeconds; + HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + } + else + { + throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", OverrideHost, TestResult.FailureReason)); + } + } + else + { + if (DiscoveryResponse.ServerEndPoints.empty()) + { + throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", DiscoveryHost)); + } + + for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : DiscoveryResponse.ServerEndPoints) + { + if (!ServerEndpoint.BaseUrl.empty()) + { + if (Verbose) + { + ZEN_INFO("Testing server endpoint at '{}/health/live'. Assume http2: {}", + ServerEndpoint.BaseUrl, + ServerEndpoint.AssumeHttp2); + } + + if (JupiterEndpointTestResult TestResult = + TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose); + TestResult.Success) + { + if (Verbose) + { + ZEN_INFO("Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); + } + + HostUrl = ServerEndpoint.BaseUrl; + HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; + HostName = ServerEndpoint.Name; + HostLatencySec = TestResult.LatencySeconds; + HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + break; + } + else + { + ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason); + } + } + } + if (HostUrl.empty()) + { + throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}", + DiscoveryResponse.ServerEndPoints.size(), + DiscoveryHost)); + } + } + if (ZenCacheHost.empty()) + { + if (AllowZenCacheDiscovery) + { + for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : DiscoveryResponse.CacheEndPoints) + { + if (!CacheEndpoint.BaseUrl.empty()) + { + if (Verbose) + { + ZEN_INFO("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", + CacheEndpoint.BaseUrl, + CacheEndpoint.AssumeHttp2); + } + + if (ZenCacheEndpointTestResult TestResult = + TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose); + TestResult.Success) + { + if (Verbose) + { + ZEN_INFO("Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); + } + + CacheUrl = CacheEndpoint.BaseUrl; + CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; + CacheName = CacheEndpoint.Name; + CacheLatencySec = TestResult.LatencySeconds; + CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + break; + } + } + } + } + if (CacheUrl.empty() && AllowLocalZenCache) + { + ZenServerState State; + if (State.InitializeReadOnly()) + { + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + if (CacheUrl.empty()) + { + std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); + if (ZenCacheEndpointTestResult TestResult = + TestZenCacheEndpoint(ZenServerLocalHostUrl, /*AssumeHttp2*/ false, ClientSettings.Verbose); + TestResult.Success) + { + CacheUrl = ZenServerLocalHostUrl; + CacheAssumeHttp2 = false; + CacheName = "localhost"; + CacheLatencySec = TestResult.LatencySeconds; + } + } + }); + } + } + } + else + { + if (Verbose) + { + ZEN_INFO("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false); + } + if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose); + TestResult.Success) + { + CacheUrl = ZenCacheHost; + CacheName = GetHostNameFromUrl(ZenCacheHost); + CacheLatencySec = TestResult.LatencySeconds; + CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + } + else + { + ZEN_WARN("Unable to reach cache host {}. Reason: {}", ZenCacheHost, TestResult.FailureReason); + } + } + + return BuildStorageResolveResult{ + .Cloud = {.Address = HostUrl, + .Name = HostName, + .AssumeHttp2 = HostAssumeHttp2, + .LatencySec = HostLatencySec, + .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = HostMaxRangeCountPerRequest}}, + .Cache = {.Address = CacheUrl, + .Name = CacheName, + .AssumeHttp2 = CacheAssumeHttp2, + .LatencySec = CacheLatencySec, + .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = CacheMaxRangeCountPerRequest}}}; +} + +} // namespace zen |