builds url discovery (#334)

- Feature: Added `--host` option to use Jupiters list of cloud host and zen servers to resolve best hosts
- Feature: Use local zenserver as builds cache if it has the `builds` service enabled and `--cloud-discovery-host` is provided and no remote zenserver cache hosts can be found
- Improvement: Added `--override-host` option as a replacement for `--url` (`--url` still works, but `--override-host` is preferred)

1 files changed, 204 insertions, 43 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 689b29db5..1c24a3c5b 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -883,6 +883,7 @@ namespace {
 		std::string						   StorageName;
 		std::unique_ptr<HttpClient>		   CacheHttp;
 		std::unique_ptr<BuildStorageCache> BuildCacheStorage;
+		std::string						   CacheName;
 	};
 
 	std::vector<uint32_t> CalculateAbsoluteChunkOrders(const std::span<const IoHash>						   LocalChunkHashes,
@@ -8348,7 +8349,14 @@ BuildsCommand::BuildsCommand()
 	auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) {
 		AddAuthOptions(Ops);
 
-		Ops.add_option("cloud build", "", "url", "Cloud Builds URL", cxxopts::value(m_BuildsUrl), "<url>");
+		Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>");
+		Ops.add_option("cloud build",
+					   "",
+					   "url",
+					   "Cloud Builds host url (legacy - use --override-host)",
+					   cxxopts::value(m_OverrideHost),
+					   "<url>");
+		Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>");
 		Ops.add_option("cloud build",
 					   "",
 					   "assume-http2",
@@ -8657,7 +8665,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	}
 
 	auto ParseStorageOptions = [&]() {
-		if (!m_BuildsUrl.empty())
+		if (!m_OverrideHost.empty() || !m_Host.empty())
 		{
 			if (!m_StoragePath.empty())
 			{
@@ -8676,7 +8684,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 
 	std::unique_ptr<AuthMgr> Auth;
-	HttpClientSettings		 ClientSettings{.AssumeHttp2 = m_AssumeHttp2, .AllowResume = true, .RetryCount = 2};
+	HttpClientSettings		 ClientSettings{.LogCategory = "httpbuildsclient",
+											.AssumeHttp2 = m_AssumeHttp2,
+											.AllowResume = true,
+											.RetryCount	 = 2};
 
 	auto CreateAuthMgr = [&]() {
 		if (!Auth)
@@ -8762,7 +8773,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth);
 		}
 
-		if (!m_BuildsUrl.empty() && !ClientSettings.AccessTokenProvider)
+		if (!ClientSettings.AccessTokenProvider)
 		{
 			ZEN_CONSOLE("Warning: No auth provider given, attempting operation without credentials.");
 		}
@@ -8781,21 +8792,176 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 		StorageInstance Result;
 
+		std::string BuildStorageName = ZEN_CLOUD_STORAGE;
+		std::string BuildCacheName;
+		bool		CacheAssumeHttp2 = false;
 		std::string StorageDescription;
 		std::string CacheDescription;
 
-		if (!m_BuildsUrl.empty())
+		if (!m_Host.empty() || !m_OverrideHost.empty())
 		{
 			ParseAuthOptions();
-			Result.BuildStorageHttp = std::make_unique<HttpClient>(m_BuildsUrl, ClientSettings);
-			StorageDescription		= fmt::format("cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
-											  m_BuildsUrl,
+		}
+
+		std::string CloudHost;
+
+		if (!m_Host.empty())
+		{
+			if (m_OverrideHost.empty() || m_ZenCacheHost.empty())
+			{
+				HttpClient			 DiscoveryHttpClient(m_Host, ClientSettings);
+				HttpClient::Response ServerInfoResponse =
+					DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON));
+				if (!ServerInfoResponse.IsSuccess())
+				{
+					throw std::runtime_error(fmt::format("Failed to get list of servers from discovery url '{}'. Reason: '{}'",
+														 m_Host,
+														 ServerInfoResponse.ErrorMessage("")));
+				}
+
+				std::string_view JsonResponse		= ServerInfoResponse.AsText();
+				CbObject		 ResponseObjectView = LoadCompactBinaryFromJson(JsonResponse).AsObject();
+
+				if (m_OverrideHost.empty())
+				{
+					CbArrayView	  ServerEndpointsArray = ResponseObjectView["serverEndpoints"sv].AsArrayView();
+					std::uint64_t ServerCount		   = ServerEndpointsArray.Num();
+					if (ServerCount == 0)
+					{
+						throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host));
+					}
+					for (CbFieldView ServerEndpointView : ServerEndpointsArray)
+					{
+						CbObjectView ServerEndpointObject = ServerEndpointView.AsObjectView();
+
+						std::string_view BaseUrl = ServerEndpointObject["baseUrl"sv].AsString();
+						if (!BaseUrl.empty())
+						{
+							const bool		 AssumeHttp2 = ServerEndpointObject["assumeHttp2"sv].AsBool(false);
+							std::string_view Name		 = ServerEndpointObject["name"sv].AsString();
+
+							HttpClientSettings TestClientSettings{.LogCategory	  = "httpbuildsclient",
+																  .ConnectTimeout = std::chrono::milliseconds{1000},
+																  .Timeout		  = std::chrono::milliseconds{2000},
+																  .AssumeHttp2	  = AssumeHttp2,
+																  .AllowResume	  = true,
+																  .RetryCount	  = 0};
+
+							HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
+							HttpClient::Response TestResponse = TestHttpClient.Get("/health/live");
+							if (TestResponse.IsSuccess())
+							{
+								CloudHost		 = BaseUrl;
+								m_AssumeHttp2	 = AssumeHttp2;
+								BuildStorageName = Name;
+								break;
+							}
+						}
+					}
+					if (CloudHost.empty())
+					{
+						throw std::runtime_error(
+							fmt::format("Failed to find any usable builds hosts out of {} using {}", ServerCount, m_Host));
+					}
+				}
+
+				auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> bool {
+					HttpClientSettings	 TestClientSettings{.LogCategory	= "httpcacheclient",
+															.ConnectTimeout = std::chrono::milliseconds{1000},
+															.Timeout		= std::chrono::milliseconds{2000},
+															.AssumeHttp2	= AssumeHttp2,
+															.AllowResume	= true,
+															.RetryCount		= 0};
+					HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
+					HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds");
+					if (TestResponse.IsSuccess())
+					{
+						return true;
+					}
+					return false;
+				};
+
+				if (m_ZenCacheHost.empty())
+				{
+					CbArrayView	  CacheEndpointsArray = ResponseObjectView["cacheEndpoints"sv].AsArrayView();
+					std::uint64_t CacheCount		  = CacheEndpointsArray.Num();
+					for (CbFieldView CacheEndpointView : CacheEndpointsArray)
+					{
+						CbObjectView CacheEndpointObject = CacheEndpointView.AsObjectView();
+
+						std::string_view BaseUrl = CacheEndpointObject["baseUrl"sv].AsString();
+						if (!BaseUrl.empty())
+						{
+							const bool		 AssumeHttp2 = CacheEndpointObject["assumeHttp2"sv].AsBool(false);
+							std::string_view Name		 = CacheEndpointObject["name"sv].AsString();
+
+							if (TestCacheEndpoint(BaseUrl, AssumeHttp2))
+							{
+								m_ZenCacheHost	 = BaseUrl;
+								CacheAssumeHttp2 = AssumeHttp2;
+								BuildCacheName	 = Name;
+								break;
+							}
+						}
+					}
+					if (m_ZenCacheHost.empty())
+					{
+						ZenServerState State;
+						if (State.InitializeReadOnly())
+						{
+							State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
+								if (m_ZenCacheHost.empty())
+								{
+									std::string ZenServerLocalHostUrl =
+										fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load());
+									if (TestCacheEndpoint(ZenServerLocalHostUrl, false))
+									{
+										m_ZenCacheHost	 = ZenServerLocalHostUrl;
+										CacheAssumeHttp2 = false;
+										BuildCacheName	 = "localhost";
+									}
+								}
+							});
+						}
+						if (m_ZenCacheHost.empty())
+						{
+							ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host);
+						}
+					}
+				}
+				else if (TestCacheEndpoint(m_ZenCacheHost, false))
+				{
+					std::string::size_type HostnameStart  = 0;
+					std::string::size_type HostnameLength = std::string::npos;
+					if (auto StartPos = m_ZenCacheHost.find("//"); StartPos != std::string::npos)
+					{
+						HostnameStart = StartPos + 2;
+					}
+					if (auto EndPos = m_ZenCacheHost.find("/", HostnameStart); EndPos != std::string::npos)
+					{
+						HostnameLength = EndPos - HostnameStart;
+					}
+					BuildCacheName = m_ZenCacheHost.substr(HostnameStart, HostnameLength);
+				}
+			}
+		}
+		else
+		{
+			CloudHost = m_OverrideHost;
+		}
+
+		if (!CloudHost.empty())
+		{
+			Result.BuildStorageHttp = std::make_unique<HttpClient>(CloudHost, ClientSettings);
+			StorageDescription		= fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
+											  BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName),
+											  CloudHost,
 											  Result.BuildStorageHttp->GetSessionId(),
 											  m_Namespace,
 											  m_Bucket);
 			Result.BuildStorage =
 				CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage");
-			Result.StorageName = ZEN_CLOUD_STORAGE;
+			Result.StorageName = BuildStorageName;
 		}
 		else if (!m_StoragePath.empty())
 		{
@@ -8810,39 +8976,39 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		}
 		if (!m_ZenCacheHost.empty())
 		{
-			Result.CacheHttp = std::make_unique<HttpClient>(m_ZenCacheHost,
+			Result.CacheHttp		 = std::make_unique<HttpClient>(m_ZenCacheHost,
 															HttpClientSettings{.LogCategory	   = "httpcacheclient",
-																			   .ConnectTimeout = std::chrono::milliseconds{3000},
-																			   .Timeout		   = std::chrono::milliseconds{30000},
-																			   .AssumeHttp2	   = false,
-																			   .AllowResume	   = true,
-																			   .RetryCount	   = 0});
-			if (Result.CacheHttp->Get("/health").IsSuccess())
+																					   .ConnectTimeout = std::chrono::milliseconds{3000},
+																					   .Timeout		   = std::chrono::milliseconds{30000},
+																					   .AssumeHttp2	   = CacheAssumeHttp2,
+																					   .AllowResume	   = true,
+																					   .RetryCount	   = 0});
+			Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp,
+																  StorageCacheStats,
+																  m_Namespace,
+																  m_Bucket,
+																  TempPath / "zencache",
+																  m_PrimeCacheOnly);
+			CacheDescription		 = fmt::format("Zen {}{}. SessionId: '{}'",
+										   BuildCacheName.empty() ? "" : fmt::format("{}, ", BuildCacheName),
+										   m_ZenCacheHost,
+										   Result.CacheHttp->GetSessionId());
+
+			if (!m_Namespace.empty())
 			{
-				Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp,
-																	  StorageCacheStats,
-																	  m_Namespace,
-																	  m_Bucket,
-																	  TempPath / "zencache",
-																	  m_PrimeCacheOnly);
-				CacheDescription		 = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId());
-				if (!m_Namespace.empty())
-				{
-					CacheDescription += fmt::format(" {}.", m_Namespace);
-				}
-				if (!m_Bucket.empty())
-				{
-					CacheDescription += fmt::format(" {}.", m_Bucket);
-				}
+				CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
 			}
-			else
+			if (!m_Bucket.empty())
 			{
-				Result.CacheHttp.reset();
+				CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
 			}
+			Result.CacheName = BuildCacheName;
+		}
+		ZEN_CONSOLE("Remote: {}", StorageDescription);
+		if (!Result.CacheName.empty())
+		{
+			ZEN_CONSOLE("Cache : {}", CacheDescription);
 		}
-		ZEN_CONSOLE("Remote: {}.{}",
-					StorageDescription,
-					CacheDescription.empty() ? std::string("") : fmt::format(" Remote Cache: {}", CacheDescription));
 		return Result;
 	};
 
@@ -9272,7 +9438,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath);
 
-			if (m_BuildsUrl.empty() && StoragePath.empty())
+			if (m_OverrideHost.empty() && StoragePath.empty())
 			{
 				StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
 				CreateDirectories(StoragePath);
@@ -9280,7 +9446,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				m_StoragePath = StoragePath.generic_string();
 			}
 			auto _ = MakeGuard([&]() {
-				if (m_BuildsUrl.empty() && StoragePath.empty())
+				if (m_OverrideHost.empty() && StoragePath.empty())
 				{
 					DeleteDirectories(StoragePath);
 				}
@@ -9632,11 +9798,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		{
 			ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
 
-			if (m_BuildsUrl.empty() && m_StoragePath.empty())
-			{
-				throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help()));
-			}
-
 			if (m_BuildId.empty())
 			{
 				throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help()));