add host discovery and zen cache support for oplog import (#601)

* add host discovery and zen cache support for oplog import

1 files changed, 29 insertions, 13 deletions

diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index 1c6b5d6b0..eef9fa7e6 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -245,7 +245,8 @@ namespace {
 		std::string							Description;
 	};
 
-	CreateRemoteStoreResult CreateRemoteStore(CbObjectView				   Params,
+	CreateRemoteStoreResult CreateRemoteStore(LoggerRef					   InLog,
+											  CbObjectView				   Params,
 											  AuthMgr&					   AuthManager,
 											  size_t					   MaxBlockSize,
 											  size_t					   MaxChunkEmbedSize,
@@ -253,6 +254,8 @@ namespace {
 	{
 		ZEN_MEMSCOPE(GetProjectHttpTag());
 
+		auto Log = [InLog]() { return InLog; };
+
 		using namespace std::literals;
 
 		std::shared_ptr<RemoteProjectStore> RemoteStore;
@@ -280,7 +283,7 @@ namespace {
 				std::string(OptionalBaseName),
 				ForceDisableBlocks,
 				ForceEnableTempBlocks};
-			RemoteStore = CreateFileRemoteStore(Options);
+			RemoteStore = CreateFileRemoteStore(Log(), Options);
 		}
 
 		if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud)
@@ -372,7 +375,7 @@ namespace {
 				ForceDisableBlocks,
 				ForceDisableTempBlocks,
 				AssumeHttp2};
-			RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+			RemoteStore = CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
 		}
 
 		if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -393,19 +396,24 @@ namespace {
 				std::string(Url),
 				std::string(Project),
 				std::string(Oplog)};
-			RemoteStore = CreateZenRemoteStore(Options, TempFilePath);
+			RemoteStore = CreateZenRemoteStore(Log(), Options, TempFilePath);
 		}
 
 		if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds)
 		{
-			std::string_view BuildsServiceUrl = Builds["url"sv].AsString();
-			if (BuildsServiceUrl.empty())
+			std::string_view BuildsServiceHost		   = Builds["url"sv].AsString();
+			std::string_view BuildsServiceOverrideHost = Builds["override-host"sv].AsString();
+			if (BuildsServiceHost.empty() && BuildsServiceOverrideHost.empty())
 			{
-				return {nullptr, "Missing service url"};
+				return {nullptr, "Missing service url or host"};
 			}
 
-			std::string		 Url	   = UrlDecode(BuildsServiceUrl);
-			std::string_view Namespace = Builds["namespace"sv].AsString();
+			std::string_view BuildsZenCacheHost = Builds["zencachehost"sv].AsString();
+
+			std::string		 Host		  = UrlDecode(BuildsServiceHost);
+			std::string		 OverrideHost = UrlDecode(BuildsServiceOverrideHost);
+			std::string		 ZenHost	  = UrlDecode(BuildsZenCacheHost);
+			std::string_view Namespace	  = Builds["namespace"sv].AsString();
 			if (Namespace.empty())
 			{
 				return {nullptr, "Missing namespace"};
@@ -462,7 +470,9 @@ namespace {
 
 			BuildsRemoteStoreOptions Options = {
 				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-				Url,
+				Host,
+				OverrideHost,
+				ZenHost,
 				std::string(Namespace),
 				std::string(Bucket),
 				BuildId,
@@ -474,7 +484,13 @@ namespace {
 				ForceDisableTempBlocks,
 				AssumeHttp2,
 				MetaData};
-			RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+			RemoteStore = CreateJupiterBuildsRemoteStore(Log(),
+														 Options,
+														 TempFilePath,
+														 /*Quiet*/ false,
+														 /*Unattended*/ false,
+														 /*Hidden*/ true,
+														 GetTinyWorkerPool(EWorkloadType::Background));
 		}
 
 		if (!RemoteStore)
@@ -2623,7 +2639,7 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 				bool		 CleanOplog				  = Params["clean"].AsBool(false);
 
 				CreateRemoteStoreResult RemoteStoreResult =
-					CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath());
+					CreateRemoteStore(Log(), Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath());
 
 				if (RemoteStoreResult.Store == nullptr)
 				{
@@ -2681,7 +2697,7 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 				bool		 EmbedLooseFile			  = Params["embedloosefiles"sv].AsBool(false);
 
 				CreateRemoteStoreResult RemoteStoreResult =
-					CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath());
+					CreateRemoteStore(Log(), Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath());
 
 				if (RemoteStoreResult.Store == nullptr)
 				{