Change curl defaults on MacOS (#645)

* changed curl config to match the default from vcpkg (i.e `CURL_CA_FALLBACK=ON`) 
* disables use of Secure Transport for Mac since it's deprecated
* Also worked around an issue (with `CURL_CA_BUNDLE`) where cross compiling curl on Mac would not configure curl in the same way as when compiling natively. This meant builds would not download on ARM macs when the CI build machine architecture was x86. The workaround should be redundant if we upgrade to 8.17 and use Apple SecTrust for cert validation. This should happen soon.
* Also added various verbose logging to facilitate trouble shooting

7 files changed, 86 insertions, 19 deletions

diff --git a/src/zenremotestore/builds/buildstoragecache.cpp b/src/zenremotestore/builds/buildstoragecache.cpp
index d7442d7c5..39f2b8373 100644
--- a/src/zenremotestore/builds/buildstoragecache.cpp
+++ b/src/zenremotestore/builds/buildstoragecache.cpp
@@ -411,14 +411,15 @@ CreateZenBuildStorageCache(HttpClient&					  HttpClient,
 }
 
 ZenCacheEndpointTestResult
-TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2)
+TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose)
 {
 	HttpClientSettings	 TestClientSettings{.LogCategory	= "httpcacheclient",
 											.ConnectTimeout = std::chrono::milliseconds{1000},
 											.Timeout		= std::chrono::milliseconds{2000},
 											.AssumeHttp2	= AssumeHttp2,
 											.AllowResume	= true,
-											.RetryCount		= 0};
+											.RetryCount		= 0,
+											.Verbose		= HttpVerbose};
 	HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
 	HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds");
 	if (TestResponse.IsSuccess())
diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp
index 3680d1d83..3af323628 100644
--- a/src/zenremotestore/builds/buildstorageutil.cpp
+++ b/src/zenremotestore/builds/buildstorageutil.cpp
@@ -13,13 +13,31 @@
 #include <zenutil/zenserverprocess.h>
 
 namespace zen {
+namespace {
+	std::string ConnectionSettingsToString(const HttpClientSettings& ClientSettings)
+	{
+		ExtendableStringBuilder<128> SB;
+		SB << "\n    LogCategory:                 " << ClientSettings.LogCategory;
+		SB << "\n    ConnectTimeout:              " << ClientSettings.ConnectTimeout.count() << " ms";
+		SB << "\n    Timeout:                     " << ClientSettings.Timeout.count() << " ms";
+		SB << "\n    AccessTokenProvider:         " << ClientSettings.AccessTokenProvider.has_value();
+		SB << "\n    AssumeHttp2:                 " << ClientSettings.AssumeHttp2;
+		SB << "\n    AllowResume:                 " << ClientSettings.AllowResume;
+		SB << "\n    RetryCount:                  " << ClientSettings.RetryCount;
+		SB << "\n    SessionId:                   " << ClientSettings.SessionId.ToString();
+		SB << "\n    Verbose:                     " << ClientSettings.Verbose;
+		SB << "\n    MaximumInMemoryDownloadSize: " << ClientSettings.MaximumInMemoryDownloadSize;
+		return SB.ToString();
+	}
+}  // namespace
 
 BuildStorageResolveResult
 ResolveBuildStorage(const HttpClientSettings& ClientSettings,
 					std::string_view		  Host,
 					std::string_view		  OverrideHost,
 					std::string_view		  ZenCacheHost,
-					ZenCacheResolveMode		  ZenResolveMode)
+					ZenCacheResolveMode		  ZenResolveMode,
+					bool					  Verbose)
 {
 	bool AllowZenCacheDiscovery = ZenResolveMode == ZenCacheResolveMode::Discovery || ZenResolveMode == ZenCacheResolveMode::All;
 	bool AllowLocalZenCache		= ZenResolveMode == ZenCacheResolveMode::LocalHost || ZenResolveMode == ZenCacheResolveMode::All;
@@ -55,13 +73,29 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
 
 	if (OverrideHost.empty() || (ZenCacheHost.empty() && AllowZenCacheDiscovery))
 	{
+		if (Verbose)
+		{
+			ZEN_CONSOLE("Querying servers at '{}/api/v1/status/servers'\n  Connection settings:{}",
+						DiscoveryHost,
+						ConnectionSettingsToString(ClientSettings));
+		}
+
 		DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings);
 	}
 
 	if (!OverrideHost.empty())
 	{
-		if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2); TestResult.Success)
+		if (Verbose)
+		{
+			ZEN_CONSOLE("Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2);
+		}
+		if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2, ClientSettings.Verbose);
+			TestResult.Success)
 		{
+			if (Verbose)
+			{
+				ZEN_CONSOLE("Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost);
+			}
 			HostUrl	 = OverrideHost;
 			HostName = GetHostNameFromUrl(OverrideHost);
 		}
@@ -81,9 +115,19 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
 		{
 			if (!ServerEndpoint.BaseUrl.empty())
 			{
-				if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2);
+				if (Verbose)
+				{
+					ZEN_CONSOLE("Testing server endpoint at '{}/health/live'. Assume http2: {}",
+								ServerEndpoint.BaseUrl,
+								ServerEndpoint.AssumeHttp2);
+				}
+
+				if (JupiterEndpointTestResult TestResult =
+						TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose);
 					TestResult.Success)
 				{
+					ZEN_CONSOLE("Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl);
+
 					HostUrl			= ServerEndpoint.BaseUrl;
 					HostAssumeHttp2 = ServerEndpoint.AssumeHttp2;
 					HostName		= ServerEndpoint.Name;
@@ -110,9 +154,19 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
 			{
 				if (!CacheEndpoint.BaseUrl.empty())
 				{
-					if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2);
+					if (Verbose)
+					{
+						ZEN_CONSOLE("Testing cache endpoint at '{}/status/builds'. Assume http2: {}",
+									CacheEndpoint.BaseUrl,
+									CacheEndpoint.AssumeHttp2);
+					}
+
+					if (ZenCacheEndpointTestResult TestResult =
+							TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose);
 						TestResult.Success)
 					{
+						ZEN_CONSOLE("Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl);
+
 						CacheUrl		 = CacheEndpoint.BaseUrl;
 						CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2;
 						CacheName		 = CacheEndpoint.Name;
@@ -130,7 +184,9 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
 					if (CacheUrl.empty())
 					{
 						std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load());
-						if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenServerLocalHostUrl, false); TestResult.Success)
+						if (ZenCacheEndpointTestResult TestResult =
+								TestZenCacheEndpoint(ZenServerLocalHostUrl, /*AssumeHttp2*/ false, ClientSettings.Verbose);
+							TestResult.Success)
 						{
 							CacheUrl		 = ZenServerLocalHostUrl;
 							CacheAssumeHttp2 = false;
@@ -141,14 +197,22 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
 			}
 		}
 	}
-	else if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, false); TestResult.Success)
-	{
-		CacheUrl  = ZenCacheHost;
-		CacheName = GetHostNameFromUrl(ZenCacheHost);
-	}
 	else
 	{
-		ZEN_WARN("Unable to reach cache host {}. Reason: {}", ZenCacheHost, TestResult.FailureReason);
+		if (Verbose)
+		{
+			ZEN_CONSOLE("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false);
+		}
+		if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose);
+			TestResult.Success)
+		{
+			CacheUrl  = ZenCacheHost;
+			CacheName = GetHostNameFromUrl(ZenCacheHost);
+		}
+		else
+		{
+			ZEN_WARN("Unable to reach cache host {}. Reason: {}", ZenCacheHost, TestResult.FailureReason);
+		}
 	}
 
 	return BuildStorageResolveResult{.HostUrl		  = HostUrl,
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h
index f1916da10..bb5b1c5f4 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h
@@ -67,6 +67,6 @@ struct ZenCacheEndpointTestResult
 	std::string FailureReason;
 };
 
-ZenCacheEndpointTestResult TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2);
+ZenCacheEndpointTestResult TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose);
 
 }  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h
index 3816822be..7ac1f78d4 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h
@@ -35,7 +35,8 @@ BuildStorageResolveResult ResolveBuildStorage(const HttpClientSettings& ClientSe
 											  std::string_view			Host,
 											  std::string_view			OverrideHost,
 											  std::string_view			ZenCacheHost,
-											  ZenCacheResolveMode		ZenResolveMode);
+											  ZenCacheResolveMode		ZenResolveMode,
+											  bool						Verbose);
 
 std::vector<ChunkBlockDescription> GetBlockDescriptions(OperationLogOutput&		Output,
 														BuildStorageBase&		Storage,
diff --git a/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h b/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h
index 3bbc700b8..432496bc1 100644
--- a/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h
+++ b/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h
@@ -30,6 +30,6 @@ struct JupiterEndpointTestResult
 	std::string FailureReason;
 };
 
-JupiterEndpointTestResult TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2);
+JupiterEndpointTestResult TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose);
 
 }  // namespace zen
diff --git a/src/zenremotestore/jupiter/jupiterhost.cpp b/src/zenremotestore/jupiter/jupiterhost.cpp
index 01728f38d..6b13d69a5 100644
--- a/src/zenremotestore/jupiter/jupiterhost.cpp
+++ b/src/zenremotestore/jupiter/jupiterhost.cpp
@@ -45,14 +45,15 @@ DiscoverJupiterEndpoints(std::string_view Host, const HttpClientSettings& Client
 }
 
 JupiterEndpointTestResult
-TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2)
+TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose)
 {
 	HttpClientSettings TestClientSettings{.LogCategory	  = "httpbuildsclient",
 										  .ConnectTimeout = std::chrono::milliseconds{1000},
 										  .Timeout		  = std::chrono::milliseconds{2000},
 										  .AssumeHttp2	  = AssumeHttp2,
 										  .AllowResume	  = true,
-										  .RetryCount	  = 0};
+										  .RetryCount	  = 0,
+										  .Verbose		  = HttpVerbose};
 
 	HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
 	HttpClient::Response TestResponse = TestHttpClient.Get("/health/live");
diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
index 09ffc5d71..db2192a49 100644
--- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
@@ -665,7 +665,7 @@ CreateJupiterBuildsRemoteStore(LoggerRef					   InLog,
 										  .AllowResume		   = true,
 										  .RetryCount		   = 2};
 
-		ResolveRes = ResolveBuildStorage(ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery);
+		ResolveRes = ResolveBuildStorage(ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery, /*Verbose*/ false);
 	}
 
 	HttpClientSettings ClientSettings{.LogCategory		   = "httpbuildsclient",