add host discovery and zen cache support for oplog import (#601)

* add host discovery and zen cache support for oplog import

17 files changed, 625 insertions, 332 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 9b91a9c02..143f567af 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -26,6 +26,7 @@
 #include <zenremotestore/builds/buildsavedstate.h>
 #include <zenremotestore/builds/buildstoragecache.h>
 #include <zenremotestore/builds/buildstorageoperations.h>
+#include <zenremotestore/builds/buildstorageutil.h>
 #include <zenremotestore/builds/filebuildstorage.h>
 #include <zenremotestore/builds/jupiterbuildstorage.h>
 #include <zenremotestore/chunking/chunkblock.h>
@@ -3279,168 +3280,89 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 								  bool							 BoostCacheBackgroundWorkerPool) -> StorageInstance {
 		ParseStorageOptions(RequireNamespace, RequireBucket);
 
-		StorageInstance Result;
+		HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
+										  .AssumeHttp2 = m_AssumeHttp2,
+										  .AllowResume = true,
+										  .RetryCount  = 2};
+
+		std::unique_ptr<AuthMgr> Auth;
 
-		std::string BuildStorageName = ZEN_CLOUD_STORAGE;
-		std::string BuildCacheName;
-		bool		CacheAssumeHttp2 = false;
 		std::string StorageDescription;
 		std::string CacheDescription;
 
+		StorageInstance Result;
+
 		if (!m_Host.empty() || !m_OverrideHost.empty())
 		{
 			m_AuthOptions.ParseOptions(*SubOption,
 									   m_SystemRootDir,
 									   ClientSettings,
-									   m_OverrideHost.empty() ? m_Host : m_OverrideHost,
+									   m_Host.empty() ? m_OverrideHost : m_Host,
 									   Auth,
 									   IsQuiet,
 									   /*Hidden*/ false);
-		}
 
-		std::string CloudHost;
-
-		if (!m_Host.empty())
-		{
-			if (m_OverrideHost.empty() || m_ZenCacheHost.empty())
+			BuildStorageResolveResult ResolveRes =
+				ResolveBuildStorage(ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All);
+			if (!ResolveRes.HostUrl.empty())
 			{
-				JupiterServerDiscovery Response = DiscoverJupiterEndpoints(m_Host, ClientSettings);
-
-				if (m_OverrideHost.empty())
+				Result.BuildStorageHttp =
+					std::make_unique<HttpClient>(ResolveRes.HostUrl, ClientSettings, []() { return AbortFlag.load(); });
+
+				Result.BuildStorage = CreateJupiterBuildStorage(Log(),
+																*Result.BuildStorageHttp,
+																StorageStats,
+																m_Namespace,
+																m_Bucket,
+																m_AllowRedirect,
+																TempPath / "storage");
+				Result.StorageName	= ResolveRes.HostName;
+
+				StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
+												 ResolveRes.HostName,
+												 (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl),
+												 Result.BuildStorageHttp->GetSessionId(),
+												 m_Namespace,
+												 m_Bucket);
+				;
+
+				if (!ResolveRes.CacheUrl.empty())
 				{
-					if (Response.ServerEndPoints.empty())
-					{
-						throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host));
-					}
-					for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : Response.ServerEndPoints)
+					Result.CacheHttp = std::make_unique<HttpClient>(ResolveRes.CacheUrl,
+																	HttpClientSettings{.LogCategory	   = "httpcacheclient",
+																					   .ConnectTimeout = std::chrono::milliseconds{3000},
+																					   .Timeout		   = std::chrono::milliseconds{30000},
+																					   .AssumeHttp2	   = ResolveRes.CacheAssumeHttp2,
+																					   .AllowResume	   = true,
+																					   .RetryCount	   = 0},
+																	[]() { return AbortFlag.load(); });
+					Result.BuildCacheStorage =
+						CreateZenBuildStorageCache(*Result.CacheHttp,
+												   StorageCacheStats,
+												   m_Namespace,
+												   m_Bucket,
+												   TempPath / "zencache",
+												   BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
+																				  : GetTinyWorkerPool(EWorkloadType::Background));
+					Result.CacheName = ResolveRes.CacheName;
+
+					CacheDescription =
+						fmt::format("Zen {}{}. SessionId: '{}'",
+									ResolveRes.CacheName,
+									(ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl),
+									Result.CacheHttp->GetSessionId());
+					;
+					if (!m_Namespace.empty())
 					{
-						if (!ServerEndpoint.BaseUrl.empty())
-						{
-							if (JupiterEndpointTestResult TestResult =
-									TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2);
-								TestResult.Success)
-							{
-								CloudHost		 = ServerEndpoint.BaseUrl;
-								m_AssumeHttp2	 = ServerEndpoint.AssumeHttp2;
-								BuildStorageName = ServerEndpoint.Name;
-								break;
-							}
-							else
-							{
-								ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason);
-							}
-						}
+						CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
 					}
-					if (CloudHost.empty())
+					if (!m_Bucket.empty())
 					{
-						throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}",
-															 Response.ServerEndPoints.size(),
-															 m_Host));
+						CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
 					}
 				}
-				else if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(m_OverrideHost, m_AssumeHttp2); TestResult.Success)
-				{
-					CloudHost = m_OverrideHost;
-				}
-				else
-				{
-					throw std::runtime_error(
-						fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.FailureReason));
-				}
-
-				if (m_ZenCacheHost.empty())
-				{
-					for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : Response.CacheEndPoints)
-					{
-						if (!CacheEndpoint.BaseUrl.empty())
-						{
-							if (ZenCacheEndpointTestResult TestResult =
-									TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2);
-								TestResult.Success)
-							{
-								m_ZenCacheHost	 = CacheEndpoint.BaseUrl;
-								CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2;
-								BuildCacheName	 = CacheEndpoint.Name;
-								break;
-							}
-						}
-					}
-					if (m_ZenCacheHost.empty())
-					{
-						ZenServerState State;
-						if (State.InitializeReadOnly())
-						{
-							State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
-								if (m_ZenCacheHost.empty())
-								{
-									std::string ZenServerLocalHostUrl =
-										fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load());
-									if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenServerLocalHostUrl, false);
-										TestResult.Success)
-									{
-										m_ZenCacheHost	 = ZenServerLocalHostUrl;
-										CacheAssumeHttp2 = false;
-										BuildCacheName	 = "localhost";
-									}
-								}
-							});
-						}
-						if (m_ZenCacheHost.empty() && !IsQuiet)
-						{
-							ZEN_CONSOLE_WARN("Failed to find any usable cache hosts out of {} using {}",
-											 Response.CacheEndPoints.size(),
-											 m_Host);
-						}
-					}
-				}
-				else if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, false); TestResult.Success)
-				{
-					std::string::size_type HostnameStart  = 0;
-					std::string::size_type HostnameLength = std::string::npos;
-					if (auto StartPos = m_ZenCacheHost.find("//"); StartPos != std::string::npos)
-					{
-						HostnameStart = StartPos + 2;
-					}
-					if (auto EndPos = m_ZenCacheHost.find("/", HostnameStart); EndPos != std::string::npos)
-					{
-						HostnameLength = EndPos - HostnameStart;
-					}
-					BuildCacheName = m_ZenCacheHost.substr(HostnameStart, HostnameLength);
-				}
-				else
-				{
-					ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.FailureReason);
-					m_ZenCacheHost = "";
-				}
-			}
-			else if (!m_OverrideHost.empty())
-			{
-				CloudHost = m_OverrideHost;
 			}
 		}
-		else
-		{
-			CloudHost = m_OverrideHost;
-		}
-
-		if (!CloudHost.empty())
-		{
-			Result.BuildStorageHttp = std::make_unique<HttpClient>(CloudHost, ClientSettings, []() { return AbortFlag.load(); });
-			StorageDescription		= fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
-											  BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName),
-											  CloudHost,
-											  Result.BuildStorageHttp->GetSessionId(),
-											  m_Namespace,
-											  m_Bucket);
-			Result.BuildStorage		= CreateJupiterBuildStorage(Log(),
-															*Result.BuildStorageHttp,
-															StorageStats,
-															m_Namespace,
-															m_Bucket,
-															m_AllowRedirect,
-															TempPath / "storage");
-			Result.StorageName		= BuildStorageName;
-		}
 		else if (!m_StoragePath.empty())
 		{
 			StorageDescription	= fmt::format("folder {}", m_StoragePath);
@@ -3451,47 +3373,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		{
 			throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help());
 		}
-		if (!m_ZenCacheHost.empty())
-		{
-			if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, false); TestResult.Success)
-			{
-				Result.CacheHttp = std::make_unique<HttpClient>(m_ZenCacheHost,
-																HttpClientSettings{.LogCategory	   = "httpcacheclient",
-																				   .ConnectTimeout = std::chrono::milliseconds{3000},
-																				   .Timeout		   = std::chrono::milliseconds{30000},
-																				   .AssumeHttp2	   = CacheAssumeHttp2,
-																				   .AllowResume	   = true,
-																				   .RetryCount	   = 0},
-																[]() { return AbortFlag.load(); });
-				Result.BuildCacheStorage =
-					CreateZenBuildStorageCache(*Result.CacheHttp,
-											   StorageCacheStats,
-											   m_Namespace,
-											   m_Bucket,
-											   TempPath / "zencache",
-											   BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
-																			  : GetTinyWorkerPool(EWorkloadType::Background));
-				CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'",
-											   BuildCacheName.empty() ? "" : fmt::format("{}, ", BuildCacheName),
-											   m_ZenCacheHost,
-											   Result.CacheHttp->GetSessionId());
-
-				if (!m_Namespace.empty())
-				{
-					CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
-				}
-				if (!m_Bucket.empty())
-				{
-					CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
-				}
-				Result.CacheName = BuildCacheName.empty() ? m_ZenCacheHost : BuildCacheName;
-			}
-			else
-			{
-				ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.FailureReason);
-				m_ZenCacheHost = "";
-			}
-		}
+
 		if (!IsQuiet)
 		{
 			ZEN_CONSOLE("Remote: {}", StorageDescription);
diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index 3f82bf982..e400b96e7 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -19,6 +19,7 @@
 #include <zenhttp/httpclient.h>
 #include <zenhttp/httpclientauth.h>
 #include <zenhttp/httpcommon.h>
+#include <zenremotestore/builds/buildstorageutil.h>
 #include <zenremotestore/builds/jupiterbuildstorage.h>
 #include <zenremotestore/jupiter/jupiterhost.h>
 
@@ -1361,9 +1362,20 @@ ImportOplogCommand::ImportOplogCommand()
 						 "<assumehttp2>");
 
 	m_Options.add_option("", "", "cloud", "Cloud Storage URL", cxxopts::value(m_CloudUrl), "<url>");
+
 	m_Options.add_option("cloud", "", "key", "Cloud Storage key", cxxopts::value(m_CloudKey), "<key>");
 
-	m_Options.add_option("", "", "builds", "Builds Storage URL", cxxopts::value(m_BuildsUrl), "<url>");
+	m_Options.add_option("", "", "builds", "Builds Storage URL", cxxopts::value(m_BuildsHost), "<url>");
+
+	m_Options.add_option("",
+						 "",
+						 "builds-override-host",
+						 "Builds Storage override API host",
+						 cxxopts::value(m_BuildsOverrideHost),
+						 "<override-host>");
+
+	m_Options
+		.add_option("builds", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>");
 	m_Options.add_option("builds", "", "builds-id", "Builds Id", cxxopts::value(m_BuildsId), "<id>");
 
 	m_Options.add_option("", "", "zen", "Zen service upload address", cxxopts::value(m_ZenUrl), "<url>");
@@ -1421,7 +1433,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 
 	size_t TargetCount = 0;
 	TargetCount += m_CloudUrl.empty() ? 0 : 1;
-	TargetCount += m_BuildsUrl.empty() ? 0 : 1;
+	TargetCount += (m_BuildsHost.empty() && m_BuildsOverrideHost.empty()) ? 0 : 1;
 	TargetCount += m_ZenUrl.empty() ? 0 : 1;
 	TargetCount += m_FileDirectoryPath.empty() ? 0 : 1;
 	if (TargetCount == 0)
@@ -1452,7 +1464,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 		}
 	}
 
-	if (!m_BuildsUrl.empty())
+	if (!m_BuildsHost.empty() || !m_BuildsOverrideHost.empty())
 	{
 		if (m_JupiterNamespace.empty())
 		{
@@ -1572,11 +1584,13 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 				Writer.EndObject();	 // "cloud"
 				SourceDescription = fmt::format("[cloud] {}/{}/{}/{}", m_CloudUrl, m_JupiterNamespace, m_JupiterBucket, m_CloudKey);
 			}
-			if (!m_BuildsUrl.empty())
+			if (!m_BuildsHost.empty() || !m_BuildsOverrideHost.empty())
 			{
 				Writer.BeginObject("builds"sv);
 				{
-					Writer.AddString("url"sv, m_BuildsUrl);
+					Writer.AddString("url"sv, m_BuildsHost);
+					Writer.AddString("override-host"sv, m_BuildsOverrideHost);
+					Writer.AddString("zencachehost"sv, m_ZenCacheHost);
 					Writer.AddString("namespace"sv, m_JupiterNamespace);
 					Writer.AddString("bucket"sv, m_JupiterBucket);
 					Writer.AddString("buildsid"sv, m_BuildsId);
@@ -2316,27 +2330,13 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
 							   m_Quiet,
 							   /*Hidden*/ false);
 
+	BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ClientSettings, m_Host, m_OverrideHost, ""sv, ZenCacheResolveMode::Off);
+
+#if 0
 	std::string BuildStorageName = ZEN_CLOUD_STORAGE;
 
 	std::string CloudHost;
 
-	auto TestHostEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair<bool, std::string> {
-		HttpClientSettings TestClientSettings{.LogCategory	  = "httpbuildsclient",
-											  .ConnectTimeout = std::chrono::milliseconds{1000},
-											  .Timeout		  = std::chrono::milliseconds{2000},
-											  .AssumeHttp2	  = AssumeHttp2,
-											  .AllowResume	  = true,
-											  .RetryCount	  = 0};
-
-		HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
-		HttpClient::Response TestResponse = TestHttpClient.Get("/health/live");
-		if (TestResponse.IsSuccess())
-		{
-			return {true, ""};
-		}
-		return {false, TestResponse.ErrorMessage("")};
-	};
-
 	if (m_OverrideHost.empty())
 	{
 		JupiterServerDiscovery Response = DiscoverJupiterEndpoints(m_Host, ClientSettings);
@@ -2377,6 +2377,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
 	{
 		throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.FailureReason));
 	}
+#endif	// 0
 
 	Oid BuildId = Oid::TryFromHexString(m_BuildId);
 	if (BuildId == Oid::Zero)
@@ -2385,16 +2386,16 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
 	}
 
 	BuildStorage::Statistics StorageStats;
-	HttpClient				 BuildStorageHttp(CloudHost, ClientSettings);
+	HttpClient				 BuildStorageHttp(ResolveRes.HostUrl, ClientSettings);
 
 	if (!m_Quiet)
 	{
-		std::string StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
-													 BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName),
-													 CloudHost,
-													 BuildStorageHttp.GetSessionId(),
-													 m_Namespace,
-													 m_Bucket);
+		std::string StorageDescription =
+			fmt::format("Cloud {}{}. Namespace '{}', Bucket '{}'",
+						ResolveRes.HostName,
+						(ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl),
+						m_Namespace,
+						m_Bucket);
 
 		ZEN_CONSOLE("Remote: {}", StorageDescription);
 	}
diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h
index ae8233f51..58002a533 100644
--- a/src/zen/cmds/projectstore_cmd.h
+++ b/src/zen/cmds/projectstore_cmd.h
@@ -172,9 +172,11 @@ private:
 	bool		m_JupiterAssumeHttp2 = false;
 
 	std::string m_CloudUrl;
+	std::string m_ZenCacheHost;
 	std::string m_CloudKey;
 
-	std::string m_BuildsUrl;
+	std::string m_BuildsHost;
+	std::string m_BuildsOverrideHost;
 	std::string m_BuildsId;
 
 	std::string m_ZenUrl;
diff --git a/src/zenremotestore/builds/buildstoragecache.cpp b/src/zenremotestore/builds/buildstoragecache.cpp
index 2e9cf9954..d7442d7c5 100644
--- a/src/zenremotestore/builds/buildstoragecache.cpp
+++ b/src/zenremotestore/builds/buildstoragecache.cpp
@@ -359,19 +359,16 @@ public:
 		while (true)
 		{
 			intptr_t Remaining = m_PendingBackgroundWorkCount.Remaining();
-			if (UpdateCallback(Remaining))
+			if (!UpdateCallback(Remaining))
 			{
-				if (m_PendingBackgroundWorkCount.Wait(UpdateIntervalMS))
-				{
-					UpdateCallback(0);
-					return;
-				}
+				m_CancelBackgroundWork.store(true);
 			}
-			else
+			if (m_PendingBackgroundWorkCount.Wait(UpdateIntervalMS))
 			{
-				m_CancelBackgroundWork.store(true);
+				break;
 			}
 		}
+		UpdateCallback(0);
 	}
 
 private:
diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp
new file mode 100644
index 000000000..998529714
--- /dev/null
+++ b/src/zenremotestore/builds/buildstorageutil.cpp
@@ -0,0 +1,157 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/buildstorageutil.h>
+
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/jupiterbuildstorage.h>
+#include <zenremotestore/jupiter/jupiterhost.h>
+#include <zenutil/zenserverprocess.h>
+
+namespace zen {
+
+BuildStorageResolveResult
+ResolveBuildStorage(const HttpClientSettings& ClientSettings,
+					std::string_view		  Host,
+					std::string_view		  OverrideHost,
+					std::string_view		  ZenCacheHost,
+					ZenCacheResolveMode		  ZenResolveMode)
+{
+	bool AllowZenCacheDiscovery = ZenResolveMode == ZenCacheResolveMode::Discovery || ZenResolveMode == ZenCacheResolveMode::All;
+	bool AllowLocalZenCache		= ZenResolveMode == ZenCacheResolveMode::LocalHost || ZenResolveMode == ZenCacheResolveMode::All;
+
+	auto GetHostNameFromUrl = [](std::string_view Url) -> std::string_view {
+		std::string::size_type HostnameStart  = 0;
+		std::string::size_type HostnameLength = std::string::npos;
+		if (auto StartPos = Url.find("//"); StartPos != std::string::npos)
+		{
+			HostnameStart = StartPos + 2;
+		}
+		if (auto EndPos = Url.find("/", HostnameStart); EndPos != std::string::npos)
+		{
+			HostnameLength = EndPos - HostnameStart;
+		}
+		if (auto EndPos = Url.find(":", HostnameStart); EndPos != std::string::npos)
+		{
+			HostnameLength = EndPos - HostnameStart;
+		}
+		return Url.substr(HostnameStart, HostnameLength);
+	};
+
+	std::string HostUrl;
+	std::string HostName;
+
+	std::string CacheUrl;
+	std::string CacheName;
+	bool		HostAssumeHttp2	 = ClientSettings.AssumeHttp2;
+	bool		CacheAssumeHttp2 = ClientSettings.AssumeHttp2;
+
+	JupiterServerDiscovery DiscoveryResponse;
+	const std::string_view DiscoveryHost = Host.empty() ? OverrideHost : Host;
+
+	if (OverrideHost.empty() || (ZenCacheHost.empty() && AllowZenCacheDiscovery))
+	{
+		DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings);
+	}
+
+	if (!OverrideHost.empty())
+	{
+		if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2); TestResult.Success)
+		{
+			HostUrl	 = OverrideHost;
+			HostName = GetHostNameFromUrl(OverrideHost);
+		}
+		else
+		{
+			throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", OverrideHost, TestResult.FailureReason));
+		}
+	}
+	else
+	{
+		if (DiscoveryResponse.ServerEndPoints.empty())
+		{
+			throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", DiscoveryHost));
+		}
+
+		for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : DiscoveryResponse.ServerEndPoints)
+		{
+			if (!ServerEndpoint.BaseUrl.empty())
+			{
+				if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2);
+					TestResult.Success)
+				{
+					HostUrl			= ServerEndpoint.BaseUrl;
+					HostAssumeHttp2 = ServerEndpoint.AssumeHttp2;
+					HostName		= ServerEndpoint.Name;
+					break;
+				}
+				else
+				{
+					ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason);
+				}
+			}
+		}
+		if (HostUrl.empty())
+		{
+			throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}",
+												 DiscoveryResponse.ServerEndPoints.size(),
+												 DiscoveryHost));
+		}
+	}
+	if (ZenCacheHost.empty())
+	{
+		if (AllowZenCacheDiscovery)
+		{
+			for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : DiscoveryResponse.CacheEndPoints)
+			{
+				if (!CacheEndpoint.BaseUrl.empty())
+				{
+					if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2);
+						TestResult.Success)
+					{
+						CacheUrl		 = CacheEndpoint.BaseUrl;
+						CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2;
+						CacheName		 = CacheEndpoint.Name;
+						break;
+					}
+				}
+			}
+		}
+		if (CacheUrl.empty() && AllowLocalZenCache)
+		{
+			ZenServerState State;
+			if (State.InitializeReadOnly())
+			{
+				State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
+					if (CacheUrl.empty())
+					{
+						std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load());
+						if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenServerLocalHostUrl, false); TestResult.Success)
+						{
+							CacheUrl		 = ZenServerLocalHostUrl;
+							CacheAssumeHttp2 = false;
+							CacheName		 = "localhost";
+						}
+					}
+				});
+			}
+		}
+	}
+	else if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, false); TestResult.Success)
+	{
+		CacheUrl  = ZenCacheHost;
+		CacheName = GetHostNameFromUrl(ZenCacheHost);
+	}
+	else
+	{
+		ZEN_WARN("Unable to reach cache host {}. Reason: {}", ZenCacheHost, TestResult.FailureReason);
+	}
+
+	return BuildStorageResolveResult{.HostUrl		  = HostUrl,
+									 .HostName		  = HostName,
+									 .HostAssumeHttp2 = HostAssumeHttp2,
+
+									 .CacheUrl		   = CacheUrl,
+									 .CacheName		   = CacheName,
+									 .CacheAssumeHttp2 = CacheAssumeHttp2};
+}
+}  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h
new file mode 100644
index 000000000..258266a6a
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h
@@ -0,0 +1,40 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/logging.h>
+#include <zenhttp/httpclient.h>
+#include <zenremotestore/builds/buildstorage.h>
+
+namespace cxxopts {
+class Options;
+}
+
+namespace zen {
+
+struct BuildStorageResolveResult
+{
+	std::string HostUrl;
+	std::string HostName;
+	bool		HostAssumeHttp2 = false;
+
+	std::string CacheUrl;
+	std::string CacheName;
+	bool		CacheAssumeHttp2 = false;
+};
+
+enum class ZenCacheResolveMode
+{
+	Off,
+	Discovery,
+	LocalHost,
+	All
+};
+
+BuildStorageResolveResult ResolveBuildStorage(const HttpClientSettings& ClientSettings,
+											  std::string_view			Host,
+											  std::string_view			OverrideHost,
+											  std::string_view			ZenCacheHost,
+											  ZenCacheResolveMode		ZenResolveMode);
+
+}  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
index 037325ed1..ecd157af8 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
@@ -10,7 +10,9 @@ class AuthMgr;
 
 struct BuildsRemoteStoreOptions : RemoteStoreOptions
 {
-	std::string			  Url;
+	std::string			  Host;
+	std::string			  OverrideHost;
+	std::string			  ZenHost;
 	std::string			  Namespace;
 	std::string			  Bucket;
 	Oid					  BuildId;
@@ -24,10 +26,12 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions
 	IoBuffer			  MetaData;
 };
 
-std::shared_ptr<RemoteProjectStore> CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
+std::shared_ptr<RemoteProjectStore> CreateJupiterBuildsRemoteStore(LoggerRef					   InLog,
+																   const BuildsRemoteStoreOptions& Options,
 																   const std::filesystem::path&	   TempFilePath,
 																   bool							   Quiet,
 																   bool							   Unattended,
-																   bool							   Hidden);
+																   bool							   Hidden,
+																   WorkerThreadPool&			   CacheBackgroundWorkerPool);
 
 }  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h
index ff2ecb405..72544363f 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/fileremoteprojectstore.h
@@ -15,6 +15,6 @@ struct FileRemoteStoreOptions : RemoteStoreOptions
 	bool				  ForceEnableTempBlocks = false;
 };
 
-std::shared_ptr<RemoteProjectStore> CreateFileRemoteStore(const FileRemoteStoreOptions& Options);
+std::shared_ptr<RemoteProjectStore> CreateFileRemoteStore(LoggerRef InLog, const FileRemoteStoreOptions& Options);
 
 }  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h
index 13a346039..aeeb5c0d0 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/jupiterremoteprojectstore.h
@@ -24,7 +24,8 @@ struct JupiterRemoteStoreOptions : RemoteStoreOptions
 	bool				  AssumeHttp2			 = false;
 };
 
-std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options,
+std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(LoggerRef						  InLog,
+															 const JupiterRemoteStoreOptions& Options,
 															 const std::filesystem::path&	  TempFilePath,
 															 bool							  Quiet,
 															 bool							  Unattended,
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
index fbcdde955..4740e7029 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
@@ -105,6 +105,8 @@ public:
 	virtual GetKnownBlocksResult  GetKnownBlocks()										= 0;
 	virtual LoadAttachmentResult  LoadAttachment(const IoHash& RawHash)					= 0;
 	virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0;
+
+	virtual void Flush() = 0;
 };
 
 struct RemoteStoreOptions
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h
index d6b62c563..2188be838 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/zenremoteprojectstore.h
@@ -13,6 +13,8 @@ struct ZenRemoteStoreOptions : RemoteStoreOptions
 	std::string OplogId;
 };
 
-std::shared_ptr<RemoteProjectStore> CreateZenRemoteStore(const ZenRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath);
+std::shared_ptr<RemoteProjectStore> CreateZenRemoteStore(LoggerRef					  InLog,
+														 const ZenRemoteStoreOptions& Options,
+														 const std::filesystem::path& TempFilePath);
 
 }  // namespace zen
diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
index 2cc8ed4aa..1d749a17d 100644
--- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
@@ -8,8 +8,12 @@
 #include <zencore/scopeguard.h>
 
 #include <zenhttp/httpclientauth.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildstorageutil.h>
 #include <zenremotestore/builds/jupiterbuildstorage.h>
 
+#include <numeric>
+
 namespace zen {
 
 using namespace std::literals;
@@ -19,20 +23,28 @@ static const std::string_view OplogContainerPartName = "oplogcontainer"sv;
 class BuildsRemoteStore : public RemoteProjectStore
 {
 public:
-	BuildsRemoteStore(std::unique_ptr<BuildStorage::Statistics>&& BuildStorageStats,
-					  std::unique_ptr<HttpClient>&&				  BuildStorageHttp,
-					  std::unique_ptr<BuildStorage>&&			  BuildStorage,
-					  std::string_view							  Url,
-					  std::string_view							  Namespace,
-					  std::string_view							  Bucket,
-					  const Oid&								  BuildId,
-					  const IoBuffer&							  MetaData,
-					  bool										  ForceDisableBlocks,
-					  bool										  ForceDisableTempBlocks)
-	: m_BuildStorageStats(std::move(BuildStorageStats))
-	, m_BuildStorageHttp(std::move(BuildStorageHttp))
-	, m_BuildStorage(std::move(BuildStorage))
-	, m_Url(Url)
+	BuildsRemoteStore(LoggerRef					   InLog,
+					  const HttpClientSettings&	   ClientSettings,
+					  HttpClientSettings*		   OptionalCacheClientSettings,
+					  std::string_view			   HostUrl,
+					  std::string_view			   CacheUrl,
+					  const std::filesystem::path& TempFilePath,
+					  WorkerThreadPool&			   CacheBackgroundWorkerPool,
+					  std::string_view			   Namespace,
+					  std::string_view			   Bucket,
+					  const Oid&				   BuildId,
+					  const IoBuffer&			   MetaData,
+					  bool						   ForceDisableBlocks,
+					  bool						   ForceDisableTempBlocks)
+	: m_Log(InLog)
+	, m_BuildStorageHttp(HostUrl, ClientSettings)
+	, m_BuildStorage(CreateJupiterBuildStorage(Log(),
+											   m_BuildStorageHttp,
+											   m_BuildStorageStats,
+											   Namespace,
+											   Bucket,
+											   /*AllowRedirect*/ false,
+											   TempFilePath))
 	, m_Namespace(Namespace)
 	, m_Bucket(Bucket)
 	, m_BuildId(BuildId)
@@ -41,6 +53,17 @@ public:
 	, m_UseTempBlocks(!ForceDisableTempBlocks)
 	{
 		m_MetaData.MakeOwned();
+		if (OptionalCacheClientSettings)
+		{
+			ZEN_ASSERT(!CacheUrl.empty());
+			m_BuildCacheStorageHttp = std::make_unique<HttpClient>(CacheUrl, *OptionalCacheClientSettings);
+			m_BuildCacheStorage		= CreateZenBuildStorageCache(*m_BuildCacheStorageHttp,
+															 m_StorageCacheStats,
+															 Namespace,
+															 Bucket,
+															 TempFilePath,
+															 CacheBackgroundWorkerPool);
+		}
 	}
 
 	virtual RemoteStoreInfo GetInfo() const override
@@ -49,9 +72,10 @@ public:
 				.UseTempBlockFiles = m_UseTempBlocks,
 				.AllowChunking	   = true,
 				.ContainerName	   = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId),
-				.Description	   = fmt::format("[cloud] {}. SessionId: {}. {}/{}/{}"sv,
-											 m_Url,
-											 m_BuildStorageHttp->GetSessionId(),
+				.Description	   = fmt::format("[cloud] {}{}. SessionId: {}. {}/{}/{}"sv,
+											 m_BuildStorageHttp.GetBaseUri(),
+											 m_BuildCacheStorage ? fmt::format(" (Cache: {})", m_BuildCacheStorageHttp->GetBaseUri()) : ""sv,
+											 m_BuildStorageHttp.GetSessionId(),
 											 m_Namespace,
 											 m_Bucket,
 											 m_BuildId)};
@@ -60,13 +84,13 @@ public:
 	virtual Stats GetStats() const override
 	{
 		return {
-			.m_SentBytes		 = m_BuildStorageStats->TotalBytesWritten.load(),
-			.m_ReceivedBytes	 = m_BuildStorageStats->TotalBytesRead.load(),
-			.m_RequestTimeNS	 = m_BuildStorageStats->TotalRequestTimeUs.load() * 1000,
-			.m_RequestCount		 = m_BuildStorageStats->TotalRequestCount.load(),
-			.m_PeakSentBytes	 = m_BuildStorageStats->PeakSentBytes.load(),
-			.m_PeakReceivedBytes = m_BuildStorageStats->PeakReceivedBytes.load(),
-			.m_PeakBytesPerSec	 = m_BuildStorageStats->PeakBytesPerSec.load(),
+			.m_SentBytes	 = m_BuildStorageStats.TotalBytesWritten.load() + m_StorageCacheStats.TotalBytesWritten.load(),
+			.m_ReceivedBytes = m_BuildStorageStats.TotalBytesRead.load() + m_StorageCacheStats.TotalBytesRead.load(),
+			.m_RequestTimeNS = m_BuildStorageStats.TotalRequestTimeUs.load() * 1000 + m_StorageCacheStats.TotalRequestTimeUs.load() * 1000,
+			.m_RequestCount	 = m_BuildStorageStats.TotalRequestCount.load() + m_StorageCacheStats.TotalRequestCount.load(),
+			.m_PeakSentBytes = Max(m_BuildStorageStats.PeakSentBytes.load(), m_StorageCacheStats.PeakSentBytes.load()),
+			.m_PeakReceivedBytes = Max(m_BuildStorageStats.PeakReceivedBytes.load(), m_StorageCacheStats.PeakReceivedBytes.load()),
+			.m_PeakBytesPerSec	 = Max(m_BuildStorageStats.PeakBytesPerSec.load(), m_StorageCacheStats.PeakBytesPerSec.load()),
 		};
 	}
 
@@ -88,14 +112,22 @@ public:
 		catch (const HttpClientError& Ex)
 		{
 			Result.ErrorCode = MakeErrorCode(Ex);
-			Result.Reason =
-				fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+			Result.Reason	 = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'",
+										   m_BuildStorageHttp.GetBaseUri(),
+										   m_Namespace,
+										   m_Bucket,
+										   m_BuildId,
+										   Ex.what());
 		}
 		catch (const std::exception& Ex)
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason =
-				fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+			Result.Reason	 = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'",
+										   m_BuildStorageHttp.GetBaseUri(),
+										   m_Namespace,
+										   m_Bucket,
+										   m_BuildId,
+										   Ex.what());
 		}
 
 		return Result;
@@ -122,7 +154,7 @@ public:
 		{
 			Result.ErrorCode = MakeErrorCode(Ex);
 			Result.Reason	 = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
-										   m_Url,
+										   m_BuildStorageHttp.GetBaseUri(),
 										   m_Namespace,
 										   m_Bucket,
 										   m_BuildId,
@@ -133,7 +165,7 @@ public:
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 			Result.Reason	 = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
-										   m_Url,
+										   m_BuildStorageHttp.GetBaseUri(),
 										   m_Namespace,
 										   m_Bucket,
 										   m_BuildId,
@@ -168,7 +200,7 @@ public:
 					if (!m_BuildStorage->PutBlockMetadata(m_BuildId, RawHash, MetaPayload))
 					{
 						ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'",
-								 m_Url,
+								 m_BuildStorageHttp.GetBaseUri(),
 								 m_Namespace,
 								 m_Bucket,
 								 m_BuildId,
@@ -180,7 +212,7 @@ public:
 				{
 					Result.ErrorCode = MakeErrorCode(Ex);
 					Result.Reason	 = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'",
-												   m_Url,
+												   m_BuildStorageHttp.GetBaseUri(),
 												   m_Namespace,
 												   m_Bucket,
 												   m_BuildId,
@@ -190,7 +222,7 @@ public:
 				{
 					Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 					Result.Reason	 = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'",
-												   m_Url,
+												   m_BuildStorageHttp.GetBaseUri(),
 												   m_Namespace,
 												   m_Bucket,
 												   m_BuildId,
@@ -202,7 +234,7 @@ public:
 		{
 			Result.ErrorCode = MakeErrorCode(Ex);
 			Result.Reason	 = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'",
-										   m_Url,
+										   m_BuildStorageHttp.GetBaseUri(),
 										   m_Namespace,
 										   m_Bucket,
 										   m_BuildId,
@@ -212,7 +244,7 @@ public:
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 			Result.Reason	 = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'",
-										   m_Url,
+										   m_BuildStorageHttp.GetBaseUri(),
 										   m_Namespace,
 										   m_Bucket,
 										   m_BuildId,
@@ -259,7 +291,7 @@ public:
 							   : Ex.GetHttpResponseCode() != HttpResponseCode::ImATeapot ? (int)Ex.GetHttpResponseCode()
 																						 : 0;
 			Result.Reason	 = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
-										   m_Url,
+										   m_BuildStorageHttp.GetBaseUri(),
 										   m_Namespace,
 										   m_Bucket,
 										   m_BuildId,
@@ -270,7 +302,7 @@ public:
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 			Result.Reason	 = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
-										   m_Url,
+										   m_BuildStorageHttp.GetBaseUri(),
 										   m_Namespace,
 										   m_Bucket,
 										   m_BuildId,
@@ -290,7 +322,7 @@ public:
 								   : Ex.GetHttpResponseCode() != HttpResponseCode::ImATeapot ? (int)Ex.GetHttpResponseCode()
 																							 : 0;
 				Result.Reason	 = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
-											   m_Url,
+											   m_BuildStorageHttp.GetBaseUri(),
 											   m_Namespace,
 											   m_Bucket,
 											   m_BuildId,
@@ -300,7 +332,7 @@ public:
 			{
 				Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 				Result.Reason	 = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
-											   m_Url,
+											   m_BuildStorageHttp.GetBaseUri(),
 											   m_Namespace,
 											   m_Bucket,
 											   m_BuildId,
@@ -327,7 +359,7 @@ public:
 			if (!PartsObject)
 			{
 				throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv,
-													 m_Url,
+													 m_BuildStorageHttp.GetBaseUri(),
 													 m_Namespace,
 													 m_Bucket,
 													 m_BuildId));
@@ -336,7 +368,7 @@ public:
 			if (m_OplogBuildPartId == Oid::Zero)
 			{
 				throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv,
-													 m_Url,
+													 m_BuildStorageHttp.GetBaseUri(),
 													 m_Namespace,
 													 m_Bucket,
 													 m_BuildId,
@@ -349,7 +381,7 @@ public:
 		{
 			Result.ErrorCode = MakeErrorCode(Ex);
 			Result.Reason	 = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'",
-										   m_Url,
+										   m_BuildStorageHttp.GetBaseUri(),
 										   m_Namespace,
 										   m_Bucket,
 										   m_BuildId,
@@ -359,7 +391,7 @@ public:
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 			Result.Reason	 = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'",
-										   m_Url,
+										   m_BuildStorageHttp.GetBaseUri(),
 										   m_Namespace,
 										   m_Bucket,
 										   m_BuildId,
@@ -391,14 +423,22 @@ public:
 		catch (const HttpClientError& Ex)
 		{
 			Result.ErrorCode = MakeErrorCode(Ex);
-			Result.Reason =
-				fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+			Result.Reason	 = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'",
+										   m_BuildStorageHttp.GetBaseUri(),
+										   m_Namespace,
+										   m_Bucket,
+										   m_BuildId,
+										   Ex.what());
 		}
 		catch (const std::exception& Ex)
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason =
-				fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+			Result.Reason	 = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'",
+										   m_BuildStorageHttp.GetBaseUri(),
+										   m_Namespace,
+										   m_Bucket,
+										   m_BuildId,
+										   Ex.what());
 		}
 
 		return Result;
@@ -414,19 +454,45 @@ public:
 
 		try
 		{
-			Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash);
+			if (m_BuildCacheStorage)
+			{
+				IoBuffer CachedBlob = m_BuildCacheStorage->GetBuildBlob(m_BuildId, RawHash);
+				if (CachedBlob)
+				{
+					Result.Bytes = std::move(CachedBlob);
+				}
+			}
+			if (!Result.Bytes)
+			{
+				Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash);
+				if (m_BuildCacheStorage && Result.Bytes)
+				{
+					m_BuildCacheStorage->PutBuildBlob(m_BuildId,
+													  RawHash,
+													  Result.Bytes.GetContentType(),
+													  CompositeBuffer(SharedBuffer(Result.Bytes)));
+				}
+			}
 		}
 		catch (const HttpClientError& Ex)
 		{
 			Result.ErrorCode = MakeErrorCode(Ex);
-			Result.Reason =
-				fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+			Result.Reason	 = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'",
+										   m_BuildStorageHttp.GetBaseUri(),
+										   m_Namespace,
+										   m_Bucket,
+										   m_BuildId,
+										   Ex.what());
 		}
 		catch (const std::exception& Ex)
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason =
-				fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what());
+			Result.Reason	 = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'",
+										   m_BuildStorageHttp.GetBaseUri(),
+										   m_Namespace,
+										   m_Bucket,
+										   m_BuildId,
+										   Ex.what());
 		}
 
 		return Result;
@@ -437,7 +503,42 @@ public:
 		LoadAttachmentsResult Result;
 		Stopwatch			  Timer;
 		auto				  _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });
-		for (const IoHash& Hash : RawHashes)
+
+		std::vector<IoHash> AttachmentsLeftToFind = RawHashes;
+
+		if (m_BuildCacheStorage)
+		{
+			std::vector<BuildStorageCache::BlobExistsResult> ExistCheck = m_BuildCacheStorage->BlobsExists(m_BuildId, RawHashes);
+			if (ExistCheck.size() == RawHashes.size())
+			{
+				AttachmentsLeftToFind.clear();
+				for (size_t BlobIndex = 0; BlobIndex < RawHashes.size(); BlobIndex++)
+				{
+					const IoHash&							   Hash		  = RawHashes[BlobIndex];
+					const BuildStorageCache::BlobExistsResult& BlobExists = ExistCheck[BlobIndex];
+					if (BlobExists.HasBody)
+					{
+						IoBuffer CachedPayload = m_BuildCacheStorage->GetBuildBlob(m_BuildId, Hash);
+						if (CachedPayload)
+						{
+							Result.Chunks.emplace_back(
+								std::pair<IoHash, CompressedBuffer>{Hash,
+																	CompressedBuffer::FromCompressedNoValidate(std::move(CachedPayload))});
+						}
+						else
+						{
+							AttachmentsLeftToFind.push_back(Hash);
+						}
+					}
+					else
+					{
+						AttachmentsLeftToFind.push_back(Hash);
+					}
+				}
+			}
+		}
+
+		for (const IoHash& Hash : AttachmentsLeftToFind)
 		{
 			LoadAttachmentResult ChunkResult = LoadAttachment(Hash);
 			if (ChunkResult.ErrorCode)
@@ -445,12 +546,27 @@ public:
 				return LoadAttachmentsResult{ChunkResult};
 			}
 			ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(ChunkResult.ElapsedSeconds * 1000)));
+			if (m_BuildCacheStorage && ChunkResult.Bytes)
+			{
+				m_BuildCacheStorage->PutBuildBlob(m_BuildId,
+												  Hash,
+												  ChunkResult.Bytes.GetContentType(),
+												  CompositeBuffer(SharedBuffer(ChunkResult.Bytes)));
+			}
 			Result.Chunks.emplace_back(
 				std::pair<IoHash, CompressedBuffer>{Hash, CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult.Bytes))});
 		}
 		return Result;
 	}
 
+	virtual void Flush() override
+	{
+		if (m_BuildCacheStorage)
+		{
+			m_BuildCacheStorage->Flush(100, [](intptr_t) { return false; });
+		}
+	}
+
 private:
 	static int MakeErrorCode(const HttpClientError& Ex)
 	{
@@ -459,32 +575,54 @@ private:
 																		 : 0;
 	}
 
-	std::unique_ptr<BuildStorage::Statistics> m_BuildStorageStats;
-	std::unique_ptr<HttpClient>				  m_BuildStorageHttp;
-	std::unique_ptr<BuildStorage>			  m_BuildStorage;
-	const std::string						  m_Url;
-	const std::string						  m_Namespace;
-	const std::string						  m_Bucket;
-	const Oid								  m_BuildId;
-	IoBuffer								  m_MetaData;
-	Oid										  m_OplogBuildPartId = Oid::Zero;
-	const bool								  m_EnableBlocks	 = true;
-	const bool								  m_UseTempBlocks	 = true;
-	const bool								  m_AllowRedirect	 = false;
+	inline LoggerRef Log() const { return m_Log; }
+
+	LoggerRef m_Log;
+
+	BuildStorage::Statistics	  m_BuildStorageStats;
+	HttpClient					  m_BuildStorageHttp;
+	std::unique_ptr<BuildStorage> m_BuildStorage;
+
+	BuildStorageCache::Statistics	   m_StorageCacheStats;
+	std::unique_ptr<HttpClient>		   m_BuildCacheStorageHttp;
+	std::unique_ptr<BuildStorageCache> m_BuildCacheStorage;
+
+	const std::string m_Namespace;
+	const std::string m_Bucket;
+	const Oid		  m_BuildId;
+	IoBuffer		  m_MetaData;
+	Oid				  m_OplogBuildPartId = Oid::Zero;
+	const bool		  m_EnableBlocks	 = true;
+	const bool		  m_UseTempBlocks	 = true;
+	const bool		  m_AllowRedirect	 = false;
 };
 
 std::shared_ptr<RemoteProjectStore>
-CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
+CreateJupiterBuildsRemoteStore(LoggerRef					   InLog,
+							   const BuildsRemoteStoreOptions& Options,
 							   const std::filesystem::path&	   TempFilePath,
 							   bool							   Quiet,
 							   bool							   Unattended,
-							   bool							   Hidden)
+							   bool							   Hidden,
+							   WorkerThreadPool&			   CacheBackgroundWorkerPool)
 {
-	std::string Url = Options.Url;
-	if (Url.find("://"sv) == std::string::npos)
+	std::string Host = Options.Host;
+	if (!Host.empty() && Host.find("://"sv) == std::string::npos)
+	{
+		// Assume https URL
+		Host = fmt::format("https://{}"sv, Host);
+	}
+	std::string OverrideUrl = Options.OverrideHost;
+	if (!OverrideUrl.empty() && OverrideUrl.find("://"sv) == std::string::npos)
+	{
+		// Assume https URL
+		OverrideUrl = fmt::format("https://{}"sv, OverrideUrl);
+	}
+	std::string ZenHost = Options.ZenHost;
+	if (!ZenHost.empty() && ZenHost.find("://"sv) == std::string::npos)
 	{
 		// Assume https URL
-		Url = fmt::format("https://{}"sv, Url);
+		ZenHost = fmt::format("https://{}"sv, ZenHost);
 	}
 
 	// 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
@@ -503,7 +641,11 @@ CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
 	}
 	else if (!Options.OidcExePath.empty())
 	{
-		if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet, Unattended, Hidden);
+		if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath,
+																					Host.empty() ? OverrideUrl : Host,
+																					Quiet,
+																					Unattended,
+																					Hidden);
 			TokenProviderMaybe)
 		{
 			TokenProvider = TokenProviderMaybe.value();
@@ -515,25 +657,44 @@ CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
 		TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
 	}
 
+	BuildStorageResolveResult ResolveRes;
+	{
+		HttpClientSettings ClientSettings{.LogCategory		   = "httpbuildsclient",
+										  .AccessTokenProvider = TokenProvider,
+										  .AssumeHttp2		   = Options.AssumeHttp2,
+										  .AllowResume		   = true,
+										  .RetryCount		   = 2};
+
+		ResolveRes = ResolveBuildStorage(ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery);
+	}
+
 	HttpClientSettings ClientSettings{.LogCategory		   = "httpbuildsclient",
 									  .ConnectTimeout	   = std::chrono::milliseconds(2000),
 									  .Timeout			   = std::chrono::milliseconds(1800000),
 									  .AccessTokenProvider = std::move(TokenProvider),
-									  .AssumeHttp2		   = Options.AssumeHttp2,
+									  .AssumeHttp2		   = ResolveRes.HostAssumeHttp2,
 									  .AllowResume		   = true,
 									  .RetryCount		   = 4};
 
-	std::unique_ptr<BuildStorage::Statistics> BuildStorageStats(std::make_unique<BuildStorage::Statistics>());
-
-	std::unique_ptr<HttpClient> BuildStorageHttp = std::make_unique<HttpClient>(Url, ClientSettings);
+	std::unique_ptr<HttpClientSettings> CacheClientSettings;
 
-	std::unique_ptr<BuildStorage> BuildStorage =
-		CreateJupiterBuildStorage(Log(), *BuildStorageHttp, *BuildStorageStats, Options.Namespace, Options.Bucket, false, TempFilePath);
+	if (!ResolveRes.CacheUrl.empty())
+	{
+		CacheClientSettings = std::make_unique<HttpClientSettings>(HttpClientSettings{.LogCategory	  = "httpcacheclient",
+																					  .ConnectTimeout = std::chrono::milliseconds{3000},
+																					  .Timeout		  = std::chrono::milliseconds{30000},
+																					  .AssumeHttp2	  = ResolveRes.CacheAssumeHttp2,
+																					  .AllowResume	  = true,
+																					  .RetryCount	  = 0});
+	}
 
-	std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(std::move(BuildStorageStats),
-																						  std::move(BuildStorageHttp),
-																						  std::move(BuildStorage),
-																						  Url,
+	std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(InLog,
+																						  ClientSettings,
+																						  CacheClientSettings.get(),
+																						  ResolveRes.HostUrl,
+																						  ResolveRes.CacheUrl,
+																						  TempFilePath,
+																						  CacheBackgroundWorkerPool,
 																						  Options.Namespace,
 																						  Options.Bucket,
 																						  Options.BuildId,
diff --git a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp
index d6e6944f4..f49f0d72e 100644
--- a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp
@@ -17,12 +17,14 @@ using namespace std::literals;
 class LocalExportProjectStore : public RemoteProjectStore
 {
 public:
-	LocalExportProjectStore(std::string_view			 Name,
+	LocalExportProjectStore(LoggerRef					 InLog,
+							std::string_view			 Name,
 							std::string_view			 OptionalBaseName,
 							const std::filesystem::path& FolderPath,
 							bool						 ForceDisableBlocks,
 							bool						 ForceEnableTempBlocks)
-	: m_Name(Name)
+	: m_Log(InLog)
+	, m_Name(Name)
 	, m_OptionalBaseName(OptionalBaseName)
 	, m_OutputPath(FolderPath)
 	{
@@ -241,6 +243,8 @@ public:
 		return Result;
 	}
 
+	virtual void Flush() override {}
+
 private:
 	LoadContainerResult LoadContainer(const std::string& Name)
 	{
@@ -312,6 +316,10 @@ private:
 		m_RequestCount.fetch_add(1);
 	}
 
+	inline LoggerRef Log() const { return m_Log; }
+
+	LoggerRef m_Log;
+
 	const std::string			m_Name;
 	const std::string			m_OptionalBaseName;
 	const std::filesystem::path m_OutputPath;
@@ -328,9 +336,10 @@ private:
 };
 
 std::shared_ptr<RemoteProjectStore>
-CreateFileRemoteStore(const FileRemoteStoreOptions& Options)
+CreateFileRemoteStore(LoggerRef InLog, const FileRemoteStoreOptions& Options)
 {
-	std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<LocalExportProjectStore>(Options.Name,
+	std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<LocalExportProjectStore>(InLog,
+																								Options.Name,
 																								Options.OptionalBaseName,
 																								std::filesystem::path(Options.FolderPath),
 																								Options.ForceDisableBlocks,
diff --git a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp
index dda5ef99d..ca7a5b391 100644
--- a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp
@@ -18,7 +18,8 @@ using namespace std::literals;
 class JupiterRemoteStore : public RemoteProjectStore
 {
 public:
-	JupiterRemoteStore(Ref<JupiterClient>&&			InJupiterClient,
+	JupiterRemoteStore(LoggerRef					InLog,
+					   Ref<JupiterClient>&&			InJupiterClient,
 					   std::string_view				Namespace,
 					   std::string_view				Bucket,
 					   const IoHash&				Key,
@@ -26,7 +27,8 @@ public:
 					   bool							ForceDisableBlocks,
 					   bool							ForceDisableTempBlocks,
 					   const std::filesystem::path& TempFilePath)
-	: m_JupiterClient(std::move(InJupiterClient))
+	: m_Log(InLog)
+	, m_JupiterClient(std::move(InJupiterClient))
 	, m_Namespace(Namespace)
 	, m_Bucket(Bucket)
 	, m_Key(Key)
@@ -232,6 +234,8 @@ public:
 		return Result;
 	}
 
+	virtual void Flush() override {}
+
 private:
 	LoadContainerResult LoadContainer(const IoHash& Key)
 	{
@@ -323,6 +327,10 @@ private:
 		return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text};
 	}
 
+	inline LoggerRef Log() const { return m_Log; }
+
+	LoggerRef m_Log;
+
 	Ref<JupiterClient>	  m_JupiterClient;
 	const std::string	  m_Namespace;
 	const std::string	  m_Bucket;
@@ -343,7 +351,8 @@ private:
 };
 
 std::shared_ptr<RemoteProjectStore>
-CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options,
+CreateJupiterRemoteStore(LoggerRef						  InLog,
+						 const JupiterRemoteStoreOptions& Options,
 						 const std::filesystem::path&	  TempFilePath,
 						 bool							  Quiet,
 						 bool							  Unattended,
@@ -392,7 +401,8 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options,
 
 	Ref<JupiterClient> Client(new JupiterClient(ClientOptions, std::move(TokenProvider)));
 
-	std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<JupiterRemoteStore>(std::move(Client),
+	std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<JupiterRemoteStore>(InLog,
+																						   std::move(Client),
 																						   Options.Namespace,
 																						   Options.Bucket,
 																						   Options.Key,
diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp
index 727a25ba1..d316f67e5 100644
--- a/src/zenremotestore/projectstore/remoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp
@@ -3276,6 +3276,7 @@ LoadOplog(CidStore&			   ChunkStore,
 	{
 		if (CleanOplog)
 		{
+			RemoteStore.Flush();
 			if (!Oplog.Reset())
 			{
 				Result = RemoteProjectStore::Result{.ErrorCode		= gsl::narrow<int>(HttpResponseCode::InternalServerError),
@@ -3449,7 +3450,7 @@ TEST_CASE_TEMPLATE("project.store.export",
 									   /*OptionalBaseName = */ std::string(),
 									   /*.ForceDisableBlocks = */ Settings::ForceDisableBlocks,
 									   /*.ForceEnableTempBlocks = */ Settings::ForceEnableTempBlocks};
-	std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options);
+	std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Log(), Options);
 	RemoteProjectStore::RemoteStoreInfo StoreInfo	= RemoteStore->GetInfo();
 
 	uint32_t NetworkWorkerCount = Max(GetHardwareConcurrency() / 4u, 2u);
diff --git a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp
index 000901e45..e520466e0 100644
--- a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp
@@ -17,11 +17,13 @@ using namespace std::literals;
 class ZenRemoteStore : public RemoteProjectStore
 {
 public:
-	ZenRemoteStore(std::string_view				HostAddress,
+	ZenRemoteStore(LoggerRef					InLog,
+				   std::string_view				HostAddress,
 				   std::string_view				Project,
 				   std::string_view				Oplog,
 				   const std::filesystem::path& TempFilePath)
-	: m_HostAddress(HostAddress)
+	: m_Log(InLog)
+	, m_HostAddress(HostAddress)
 	, m_ProjectStoreUrl(fmt::format("{}/prj"sv, m_HostAddress))
 	, m_Project(Project)
 	, m_Oplog(Oplog)
@@ -266,6 +268,8 @@ public:
 		return Result;
 	}
 
+	virtual void Flush() override {}
+
 private:
 	void AddStats(const HttpClient::Response& Result)
 	{
@@ -302,6 +306,10 @@ private:
 		return {.ErrorCode = 0, .ElapsedSeconds = Response.ElapsedSeconds};
 	}
 
+	inline LoggerRef Log() const { return m_Log; }
+
+	LoggerRef m_Log;
+
 	const std::string			m_HostAddress;
 	const std::string			m_ProjectStoreUrl;
 	const std::string			m_Project;
@@ -320,7 +328,7 @@ private:
 };
 
 std::shared_ptr<RemoteProjectStore>
-CreateZenRemoteStore(const ZenRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath)
+CreateZenRemoteStore(LoggerRef InLog, const ZenRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath)
 {
 	std::string Url = Options.Url;
 	if (Url.find("://"sv) == std::string::npos)
@@ -329,7 +337,7 @@ CreateZenRemoteStore(const ZenRemoteStoreOptions& Options, const std::filesystem
 		Url = fmt::format("http://{}"sv, Url);
 	}
 	std::shared_ptr<RemoteProjectStore> RemoteStore =
-		std::make_shared<ZenRemoteStore>(Url, Options.ProjectId, Options.OplogId, TempFilePath);
+		std::make_shared<ZenRemoteStore>(InLog, Url, Options.ProjectId, Options.OplogId, TempFilePath);
 	return RemoteStore;
 }
 
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index 1c6b5d6b0..eef9fa7e6 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -245,7 +245,8 @@ namespace {
 		std::string							Description;
 	};
 
-	CreateRemoteStoreResult CreateRemoteStore(CbObjectView				   Params,
+	CreateRemoteStoreResult CreateRemoteStore(LoggerRef					   InLog,
+											  CbObjectView				   Params,
 											  AuthMgr&					   AuthManager,
 											  size_t					   MaxBlockSize,
 											  size_t					   MaxChunkEmbedSize,
@@ -253,6 +254,8 @@ namespace {
 	{
 		ZEN_MEMSCOPE(GetProjectHttpTag());
 
+		auto Log = [InLog]() { return InLog; };
+
 		using namespace std::literals;
 
 		std::shared_ptr<RemoteProjectStore> RemoteStore;
@@ -280,7 +283,7 @@ namespace {
 				std::string(OptionalBaseName),
 				ForceDisableBlocks,
 				ForceEnableTempBlocks};
-			RemoteStore = CreateFileRemoteStore(Options);
+			RemoteStore = CreateFileRemoteStore(Log(), Options);
 		}
 
 		if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud)
@@ -372,7 +375,7 @@ namespace {
 				ForceDisableBlocks,
 				ForceDisableTempBlocks,
 				AssumeHttp2};
-			RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+			RemoteStore = CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
 		}
 
 		if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -393,19 +396,24 @@ namespace {
 				std::string(Url),
 				std::string(Project),
 				std::string(Oplog)};
-			RemoteStore = CreateZenRemoteStore(Options, TempFilePath);
+			RemoteStore = CreateZenRemoteStore(Log(), Options, TempFilePath);
 		}
 
 		if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds)
 		{
-			std::string_view BuildsServiceUrl = Builds["url"sv].AsString();
-			if (BuildsServiceUrl.empty())
+			std::string_view BuildsServiceHost		   = Builds["url"sv].AsString();
+			std::string_view BuildsServiceOverrideHost = Builds["override-host"sv].AsString();
+			if (BuildsServiceHost.empty() && BuildsServiceOverrideHost.empty())
 			{
-				return {nullptr, "Missing service url"};
+				return {nullptr, "Missing service url or host"};
 			}
 
-			std::string		 Url	   = UrlDecode(BuildsServiceUrl);
-			std::string_view Namespace = Builds["namespace"sv].AsString();
+			std::string_view BuildsZenCacheHost = Builds["zencachehost"sv].AsString();
+
+			std::string		 Host		  = UrlDecode(BuildsServiceHost);
+			std::string		 OverrideHost = UrlDecode(BuildsServiceOverrideHost);
+			std::string		 ZenHost	  = UrlDecode(BuildsZenCacheHost);
+			std::string_view Namespace	  = Builds["namespace"sv].AsString();
 			if (Namespace.empty())
 			{
 				return {nullptr, "Missing namespace"};
@@ -462,7 +470,9 @@ namespace {
 
 			BuildsRemoteStoreOptions Options = {
 				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-				Url,
+				Host,
+				OverrideHost,
+				ZenHost,
 				std::string(Namespace),
 				std::string(Bucket),
 				BuildId,
@@ -474,7 +484,13 @@ namespace {
 				ForceDisableTempBlocks,
 				AssumeHttp2,
 				MetaData};
-			RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+			RemoteStore = CreateJupiterBuildsRemoteStore(Log(),
+														 Options,
+														 TempFilePath,
+														 /*Quiet*/ false,
+														 /*Unattended*/ false,
+														 /*Hidden*/ true,
+														 GetTinyWorkerPool(EWorkloadType::Background));
 		}
 
 		if (!RemoteStore)
@@ -2623,7 +2639,7 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 				bool		 CleanOplog				  = Params["clean"].AsBool(false);
 
 				CreateRemoteStoreResult RemoteStoreResult =
-					CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath());
+					CreateRemoteStore(Log(), Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath());
 
 				if (RemoteStoreResult.Store == nullptr)
 				{
@@ -2681,7 +2697,7 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 				bool		 EmbedLooseFile			  = Params["embedloosefiles"sv].AsBool(false);
 
 				CreateRemoteStoreResult RemoteStoreResult =
-					CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath());
+					CreateRemoteStore(Log(), Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath());
 
 				if (RemoteStoreResult.Store == nullptr)
 				{