fetch cloud oplog (#502)

- Feature: Added `zen oplog-download` command to download the oplog body of a cooked output stored in Cloud DDC
  - Oplog source is specified using one of the following options
	  - `--cloud-url` Cloud artifact URL for oplog
    - `--host` Base host to resolve download host from
    - `--override-host` Specific host to use without resolve
    - `--assume-http2` assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake
    - `--namespace` Builds Storage namespace
    - `--bucket` Builds Storage bucket
	  - `--build-id` an Oid in hex form for the source identifier to use
    - `--yes` suppress conformation query when doing output of a very large oplog to console
    - `--quiet` suppress all non-essential console output
    - `--output-path` path to oplog output, extension .json or .cb (compact binary). Default is output to console
    - `--system-dir` override default system root path
    - Authentication options
	    - Auth token
	      - `--access-token` http auth Cloud Storage access token
	      - `--access-token-env` name of environment variable that holds the Http auth Cloud Storage access token
	      - `--access-token-path` path to json file that holds the Http auth Cloud Storage access token
	      - `--oidctoken-exe-path` path to OidcToken executable
	    - OpenId authentication
	      - `--openid-provider-name` Open ID provider name
	      - `--openid-provider-url` Open ID provider url
	      - `--openid-client-id`Open ID client id
	      - `--openid-refresh-token` Open ID refresh token
	      - `--encryption-aes-key` 256 bit AES encryption key for storing OpenID credentials
	      - `--encryption-aes-iv` 128 bit AES encryption initialization vector for storing OpenID credentials
	    - OAuth authentication
	      - `--oauth-url` OAuth provier url
	      - `--oauth-clientid` OAuth client id
	      - `--oauth-clientsecret` OAuth client secret
- Bugfix: `zen print` command now properly outputs very large compact binary objects as json to console

2 files changed, 113 insertions, 0 deletions

diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index 386a91cb3..6eb3489dc 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -14,6 +14,8 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_map.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
+#include <regex>
+
 namespace zen {
 
 using namespace std::literals;
@@ -511,4 +513,49 @@ CreateJupiterBuildStorage(LoggerRef					   InLog,
 	return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, AllowRedirect, TempFolderPath);
 }
 
+bool
+ParseBuildStorageUrl(std::string_view InUrl,
+					 std::string&	  OutHost,
+					 std::string&	  OutNamespace,
+					 std::string&	  OutBucket,
+					 std::string&	  OutBuildId)
+{
+	std::string			   Url(InUrl);
+	const std::string_view ExtendedApiString = "api/v2/builds/";
+	if (auto ApiString = ToLower(Url).find(ExtendedApiString); ApiString != std::string::npos)
+	{
+		Url.erase(ApiString, ExtendedApiString.length());
+	}
+
+	const std::string ArtifactURLRegExString = R"((http[s]?:\/\/.*?)\/(.*?)\/(.*?)\/(.*))";
+	const std::regex  ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript | std::regex::icase);
+	std::match_results<std::string_view::const_iterator> MatchResults;
+	std::string_view									 UrlToParse(Url);
+	if (regex_match(begin(UrlToParse), end(UrlToParse), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5)
+	{
+		auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view {
+			ZEN_ASSERT(Index < MatchResults.size());
+
+			const auto& Match = MatchResults[Index];
+
+			return std::string_view(&*Match.first, Match.second - Match.first);
+		};
+
+		const std::string_view Host		 = GetMatch(1);
+		const std::string_view Namespace = GetMatch(2);
+		const std::string_view Bucket	 = GetMatch(3);
+		const std::string_view BuildId	 = GetMatch(4);
+
+		OutHost		 = Host;
+		OutNamespace = Namespace;
+		OutBucket	 = Bucket;
+		OutBuildId	 = BuildId;
+		return true;
+	}
+	else
+	{
+		return false;
+	}
+}
+
 }  // namespace zen
diff --git a/src/zenutil/jupiter/jupiterhost.cpp b/src/zenutil/jupiter/jupiterhost.cpp
new file mode 100644
index 000000000..d06229cbf
--- /dev/null
+++ b/src/zenutil/jupiter/jupiterhost.cpp
@@ -0,0 +1,66 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/jupiter/jupiterhost.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/fmtutils.h>
+#include <zenhttp/httpclient.h>
+
+namespace zen {
+
+JupiterServerDiscovery
+DiscoverJupiterEndpoints(std::string_view Host, const HttpClientSettings& ClientSettings)
+{
+	JupiterServerDiscovery Result;
+
+	HttpClient			 DiscoveryHttpClient(Host, ClientSettings);
+	HttpClient::Response ServerInfoResponse = DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON));
+	if (!ServerInfoResponse.IsSuccess())
+	{
+		ServerInfoResponse.ThrowError(fmt::format("Failed to get list of servers from discovery url '{}'", Host));
+	}
+	std::string_view JsonResponse	 = ServerInfoResponse.AsText();
+	CbObject		 CbPayload		 = LoadCompactBinaryFromJson(JsonResponse).AsObject();
+	CbArrayView		 ServerEndpoints = CbPayload["serverEndpoints"].AsArrayView();
+	Result.ServerEndPoints.reserve(ServerEndpoints.Num());
+
+	auto ParseEndPoints = [](CbArrayView ServerEndpoints) -> std::vector<JupiterServerDiscovery::EndPoint> {
+		std::vector<JupiterServerDiscovery::EndPoint> Result;
+
+		Result.reserve(ServerEndpoints.Num());
+		for (CbFieldView ServerEndpointView : ServerEndpoints)
+		{
+			CbObjectView ServerEndPoint = ServerEndpointView.AsObjectView();
+			Result.push_back(JupiterServerDiscovery::EndPoint{.Name		   = std::string(ServerEndPoint["name"].AsString()),
+															  .BaseUrl	   = std::string(ServerEndPoint["baseUrl"].AsString()),
+															  .AssumeHttp2 = ServerEndPoint["baseUrl"].AsBool(false)});
+		}
+		return Result;
+	};
+
+	Result.ServerEndPoints = ParseEndPoints(CbPayload["serverEndpoints"].AsArrayView());
+	Result.CacheEndPoints  = ParseEndPoints(CbPayload["cacheEndpoints"].AsArrayView());
+
+	return Result;
+}
+
+JupiterEndpointTestResult
+TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2)
+{
+	HttpClientSettings TestClientSettings{.LogCategory	  = "httpbuildsclient",
+										  .ConnectTimeout = std::chrono::milliseconds{1000},
+										  .Timeout		  = std::chrono::milliseconds{2000},
+										  .AssumeHttp2	  = AssumeHttp2,
+										  .AllowResume	  = true,
+										  .RetryCount	  = 0};
+
+	HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
+	HttpClient::Response TestResponse = TestHttpClient.Get("/health/live");
+	if (TestResponse.IsSuccess())
+	{
+		return {.Success = true};
+	}
+	return {.Success = false, .FailureReason = TestResponse.ErrorMessage("")};
+}
+
+}  // namespace zen