add fallback for zencache multirange (#816)

* clean up BuildStorageResolveResult to allow capabilities
* add check for multirange request capability
* add MaxRangeCountPerRequest capabilities
* project export tests
* add InMemoryBuildStorageCache
* progress and logging improvements
* fix ElapsedSeconds calculations in fileremoteprojectstore.cpp
* oplogs/builds test script

3 files changed, 204 insertions, 74 deletions

diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index 459e044eb..38d97765b 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -177,6 +177,14 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
 				uint64_t	 RangeLength = RangeView["length"sv].AsUInt64();
 				OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength));
 			}
+			if (OffsetAndLengthPairs.size() > MaxRangeCountPerRequestSupported)
+			{
+				return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+												   HttpContentType::kText,
+												   fmt::format("Number of ranges ({}) for blob request exceeds maximum range count {}",
+															   OffsetAndLengthPairs.size(),
+															   MaxRangeCountPerRequestSupported));
+			}
 		}
 		if (OffsetAndLengthPairs.empty())
 		{
@@ -661,6 +669,11 @@ HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request)
 	ZEN_TRACE_CPU("HttpBuildStoreService::Status");
 	CbObjectWriter Cbo;
 	Cbo << "ok" << true;
+	Cbo.BeginObject("capabilities");
+	{
+		Cbo << "maxrangecountperrequest" << MaxRangeCountPerRequestSupported;
+	}
+	Cbo.EndObject();  // capabilities
 	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h
index e10986411..5fa7cd642 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.h
+++ b/src/zenserver/storage/buildstore/httpbuildstore.h
@@ -45,6 +45,8 @@ private:
 
 	inline LoggerRef Log() { return m_Log; }
 
+	static constexpr uint32_t MaxRangeCountPerRequestSupported = 256u;
+
 	LoggerRef m_Log;
 
 	void PutBlobRequest(HttpRouterRequest& Req);
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index 2b5474d00..0ec6faea3 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -13,7 +13,12 @@
 #include <zencore/scopeguard.h>
 #include <zencore/stream.h>
 #include <zencore/trace.h>
+#include <zenhttp/httpclientauth.h>
 #include <zenhttp/packageformat.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildstorageutil.h>
+#include <zenremotestore/jupiter/jupiterhost.h>
+#include <zenremotestore/operationlogoutput.h>
 #include <zenremotestore/projectstore/buildsremoteprojectstore.h>
 #include <zenremotestore/projectstore/fileremoteprojectstore.h>
 #include <zenremotestore/projectstore/jupiterremoteprojectstore.h>
@@ -244,8 +249,22 @@ namespace {
 	{
 		std::shared_ptr<RemoteProjectStore> Store;
 		std::string							Description;
-		double								HostLatencySec	= -1.0;
-		double								CacheLatencySec = -1.0;
+		double								LatencySec				= -1.0;
+		uint64_t							MaxRangeCountPerRequest = 1;
+
+		struct Cache
+		{
+			std::unique_ptr<HttpClient>		   Http;
+			std::unique_ptr<BuildStorageCache> Cache;
+			Oid								   BuildsId = Oid::Zero;
+			std::string						   Description;
+			double							   LatencySec			   = -1.0;
+			uint64_t						   MaxRangeCountPerRequest = 1;
+			BuildStorageCache::Statistics	   Stats;
+			bool							   Populate = false;
+		};
+
+		std::unique_ptr<Cache> OptionalCache;
 	};
 
 	CreateRemoteStoreResult CreateRemoteStore(LoggerRef					   InLog,
@@ -262,9 +281,7 @@ namespace {
 
 		using namespace std::literals;
 
-		std::shared_ptr<RemoteProjectStore> RemoteStore;
-		double								HostLatencySec	= -1.0;
-		double								CacheLatencySec = -1.0;
+		CreateRemoteStoreResult Result;
 
 		if (CbObjectView File = Params["file"sv].AsObjectView(); File)
 		{
@@ -282,6 +299,9 @@ namespace {
 			bool			 ForceDisableBlocks	   = File["disableblocks"sv].AsBool(false);
 			bool			 ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false);
 
+			Result.LatencySec			   = 0;
+			Result.MaxRangeCountPerRequest = 1;
+
 			FileRemoteStoreOptions Options = {
 				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
 				FolderPath,
@@ -289,7 +309,7 @@ namespace {
 				std::string(OptionalBaseName),
 				ForceDisableBlocks,
 				ForceEnableTempBlocks};
-			RemoteStore = CreateFileRemoteStore(Log(), Options);
+			Result.Store = CreateFileRemoteStore(Log(), Options);
 		}
 
 		if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud)
@@ -367,21 +387,32 @@ namespace {
 			bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false);
 			bool AssumeHttp2			= Cloud["assumehttp2"sv].AsBool(false);
 
-			JupiterRemoteStoreOptions Options = {
-				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-				Url,
-				std::string(Namespace),
-				std::string(Bucket),
-				Key,
-				BaseKey,
-				std::string(OpenIdProvider),
-				AccessToken,
-				AuthManager,
-				OidcExePath,
-				ForceDisableBlocks,
-				ForceDisableTempBlocks,
-				AssumeHttp2};
-			RemoteStore = CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+			if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(Url, AssumeHttp2, /*Verbose*/ false); TestResult.Success)
+			{
+				Result.LatencySec			   = TestResult.LatencySeconds;
+				Result.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest;
+
+				JupiterRemoteStoreOptions Options = {
+					RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+					Url,
+					std::string(Namespace),
+					std::string(Bucket),
+					Key,
+					BaseKey,
+					std::string(OpenIdProvider),
+					AccessToken,
+					AuthManager,
+					OidcExePath,
+					ForceDisableBlocks,
+					ForceDisableTempBlocks,
+					AssumeHttp2};
+				Result.Store =
+					CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+			}
+			else
+			{
+				return {nullptr, fmt::format("Unable to connect to jupiter host '{}'", Url)};
+			}
 		}
 
 		if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -397,12 +428,13 @@ namespace {
 			{
 				return {nullptr, "Missing oplog"};
 			}
+
 			ZenRemoteStoreOptions Options = {
 				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
 				std::string(Url),
 				std::string(Project),
 				std::string(Oplog)};
-			RemoteStore = CreateZenRemoteStore(Log(), Options, TempFilePath);
+			Result.Store = CreateZenRemoteStore(Log(), Options, TempFilePath);
 		}
 
 		if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds)
@@ -475,11 +507,76 @@ namespace {
 			MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView();
 			IoBuffer   MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize());
 
+			auto EnsureHttps = [](const std::string& Host, std::string_view PreferredProtocol) {
+				if (!Host.empty() && Host.find("://"sv) == std::string::npos)
+				{
+					// Assume https URL
+					return fmt::format("{}://{}"sv, PreferredProtocol, Host);
+				}
+				return Host;
+			};
+
+			Host		 = EnsureHttps(Host, "https");
+			OverrideHost = EnsureHttps(OverrideHost, "https");
+			ZenHost		 = EnsureHttps(ZenHost, "http");
+
+			std::function<HttpClientAccessToken()> TokenProvider;
+			if (!OpenIdProvider.empty())
+			{
+				TokenProvider = httpclientauth::CreateFromOpenIdProvider(AuthManager, OpenIdProvider);
+			}
+			else if (!AccessToken.empty())
+			{
+				TokenProvider = httpclientauth::CreateFromStaticToken(AccessToken);
+			}
+			else if (!OidcExePath.empty())
+			{
+				if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(OidcExePath,
+																							Host.empty() ? OverrideHost : Host,
+																							/*Quiet*/ false,
+																							/*Unattended*/ false,
+																							/*Hidden*/ true);
+					TokenProviderMaybe)
+				{
+					TokenProvider = TokenProviderMaybe.value();
+				}
+			}
+
+			if (!TokenProvider)
+			{
+				TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(AuthManager);
+			}
+
+			BuildStorageResolveResult ResolveResult;
+			{
+				HttpClientSettings ClientSettings{.LogCategory		   = "httpbuildsclient",
+												  .AccessTokenProvider = TokenProvider,
+												  .AssumeHttp2		   = AssumeHttp2,
+												  .AllowResume		   = true,
+												  .RetryCount		   = 2};
+
+				std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(Log()));
+
+				try
+				{
+					ResolveResult = ResolveBuildStorage(*Output,
+														ClientSettings,
+														Host,
+														OverrideHost,
+														ZenHost,
+														ZenCacheResolveMode::Discovery,
+														/*Verbose*/ false);
+				}
+				catch (const std::exception& Ex)
+				{
+					return {nullptr, fmt::format("Failed resolving storage host and cache. Reason: '{}'", Ex.what())};
+				}
+			}
+			Result.LatencySec			   = ResolveResult.Cloud.LatencySec;
+			Result.MaxRangeCountPerRequest = ResolveResult.Cloud.Caps.MaxRangeCountPerRequest;
+
 			BuildsRemoteStoreOptions Options = {
 				RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
-				Host,
-				OverrideHost,
-				ZenHost,
 				std::string(Namespace),
 				std::string(Bucket),
 				BuildId,
@@ -489,30 +586,43 @@ namespace {
 				OidcExePath,
 				ForceDisableBlocks,
 				ForceDisableTempBlocks,
-				AssumeHttp2,
-				PopulateCache,
 				MetaData,
 				MaximumInMemoryDownloadSize};
-			RemoteStore = CreateJupiterBuildsRemoteStore(Log(),
-														 Options,
-														 TempFilePath,
-														 /*Quiet*/ false,
-														 /*Unattended*/ false,
-														 /*Hidden*/ true,
-														 GetTinyWorkerPool(EWorkloadType::Background),
-														 HostLatencySec,
-														 CacheLatencySec);
+			Result.Store = CreateJupiterBuildsRemoteStore(Log(), ResolveResult, std::move(TokenProvider), Options, TempFilePath);
+
+			if (!ResolveResult.Cache.Address.empty())
+			{
+				Result.OptionalCache = std::make_unique<CreateRemoteStoreResult::Cache>();
+
+				HttpClientSettings CacheClientSettings{.LogCategory					= "httpcacheclient",
+													   .ConnectTimeout				= std::chrono::milliseconds{3000},
+													   .Timeout						= std::chrono::milliseconds{30000},
+													   .AssumeHttp2					= ResolveResult.Cache.AssumeHttp2,
+													   .AllowResume					= true,
+													   .RetryCount					= 0,
+													   .MaximumInMemoryDownloadSize = MaximumInMemoryDownloadSize};
+
+				Result.OptionalCache->Http		 = std::make_unique<HttpClient>(ResolveResult.Cache.Address, CacheClientSettings);
+				Result.OptionalCache->Cache		 = CreateZenBuildStorageCache(*Result.OptionalCache->Http,
+																		  Result.OptionalCache->Stats,
+																		  Namespace,
+																		  Bucket,
+																		  TempFilePath,
+																		  GetTinyWorkerPool(EWorkloadType::Background));
+				Result.OptionalCache->BuildsId	 = BuildId;
+				Result.OptionalCache->LatencySec = ResolveResult.Cache.LatencySec;
+				Result.OptionalCache->MaxRangeCountPerRequest = ResolveResult.Cache.Caps.MaxRangeCountPerRequest;
+				Result.OptionalCache->Populate				  = PopulateCache;
+				Result.OptionalCache->Description =
+					fmt::format("[zenserver] {} namespace {} bucket {}", ResolveResult.Cache.Address, Namespace, Bucket);
+			}
 		}
-
-		if (!RemoteStore)
+		if (!Result.Store)
 		{
 			return {nullptr, "Unknown remote store type"};
 		}
 
-		return CreateRemoteStoreResult{.Store			= std::move(RemoteStore),
-									   .Description		= "",
-									   .HostLatencySec	= HostLatencySec,
-									   .CacheLatencySec = CacheLatencySec};
+		return Result;
 	}
 
 	std::pair<HttpResponseCode, std::string> ConvertResult(const RemoteProjectStore::Result& Result)
@@ -2679,38 +2789,36 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 				EPartialBlockRequestMode PartialBlockRequestMode =
 					PartialBlockRequestModeFromString(Params["partialblockrequestmode"sv].AsString("true"));
 
-				CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Log(),
-																			  Params,
-																			  m_AuthMgr,
-																			  MaxBlockSize,
-																			  MaxChunkEmbedSize,
-																			  GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
-																			  Oplog->TempPath());
+				std::shared_ptr<CreateRemoteStoreResult> RemoteStoreResult =
+					std::make_shared<CreateRemoteStoreResult>(CreateRemoteStore(Log(),
+																				Params,
+																				m_AuthMgr,
+																				MaxBlockSize,
+																				MaxChunkEmbedSize,
+																				GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
+																				Oplog->TempPath()));
 
-				if (RemoteStoreResult.Store == nullptr)
+				if (RemoteStoreResult->Store == nullptr)
 				{
-					return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description);
+					return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult->Description);
 				}
-				std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store);
-				RemoteProjectStore::RemoteStoreInfo StoreInfo	= RemoteStore->GetInfo();
 
 				JobId JobId = m_JobQueue.QueueJob(
 					fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()),
 					[this,
-					 ChunkStore		   = &m_CidStore,
-					 ActualRemoteStore = std::move(RemoteStore),
+					 RemoteStoreResult = std::move(RemoteStoreResult),
 					 Oplog,
 					 Force,
 					 IgnoreMissingAttachments,
 					 CleanOplog,
 					 PartialBlockRequestMode,
-					 HostLatencySec	 = RemoteStoreResult.HostLatencySec,
-					 CacheLatencySec = RemoteStoreResult.CacheLatencySec,
 					 BoostWorkerCount](JobContext& Context) {
-						Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}",
-														  Oplog->GetOuterProjectIdentifier(),
-														  Oplog->OplogId(),
-														  ActualRemoteStore->GetInfo().Description));
+						Context.ReportMessage(
+							fmt::format("Loading oplog '{}/{}'\n    Host: {}\n    Cache: {}",
+										Oplog->GetOuterProjectIdentifier(),
+										Oplog->OplogId(),
+										RemoteStoreResult->Store->GetInfo().Description,
+										RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Description : "<none>"));
 
 						Ref<TransferThreadWorkers> Workers = GetThreadWorkers(BoostWorkerCount, /*SingleThreaded*/ false);
 
@@ -2718,19 +2826,26 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 						WorkerThreadPool& NetworkWorkerPool = Workers->GetNetworkPool();
 
 						Context.ReportMessage(fmt::format("{}", Workers->GetWorkersInfo()));
-
-						RemoteProjectStore::Result Result	= LoadOplog(m_CidStore,
-																		*ActualRemoteStore,
-																		*Oplog,
-																		NetworkWorkerPool,
-																		WorkerPool,
-																		Force,
-																		IgnoreMissingAttachments,
-																		CleanOplog,
-																		PartialBlockRequestMode,
-																		HostLatencySec,
-																		CacheLatencySec,
-																		&Context);
+						RemoteProjectStore::Result Result	= LoadOplog(LoadOplogContext{
+							  .ChunkStore	 = m_CidStore,
+							  .RemoteStore	 = *RemoteStoreResult->Store,
+							  .OptionalCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Cache.get() : nullptr,
+							  .CacheBuildId	 = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->BuildsId : Oid::Zero,
+							  .OptionalCacheStats = RemoteStoreResult->OptionalCache ? &RemoteStoreResult->OptionalCache->Stats : nullptr,
+							  .Oplog			  = *Oplog,
+							  .NetworkWorkerPool  = NetworkWorkerPool,
+							  .WorkerPool		  = WorkerPool,
+							  .ForceDownload	  = Force,
+							  .IgnoreMissingAttachments = IgnoreMissingAttachments,
+							  .CleanOplog				= CleanOplog,
+							  .PartialBlockRequestMode	= PartialBlockRequestMode,
+							  .PopulateCache   = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Populate : false,
+							  .StoreLatencySec = RemoteStoreResult->LatencySec,
+							  .StoreMaxRangeCountPerRequest = RemoteStoreResult->MaxRangeCountPerRequest,
+							  .CacheLatencySec = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->LatencySec : -1.0,
+							  .CacheMaxRangeCountPerRequest =
+								  RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->MaxRangeCountPerRequest : 0,
+							  .OptionalJobContext = &Context});
 						auto					   Response = ConvertResult(Result);
 						ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second);
 						if (!IsHttpSuccessCode(Response.first))