// Copyright Epic Games, Inc. All Rights Reserved.

#include <zenremotestore/projectstore/buildsremoteprojectstore.h>

#include <zencore/compactbinarybuilder.h>
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
#include <zencore/scopeguard.h>

#include <zenhttp/httpclientauth.h>
#include <zenremotestore/builds/buildstoragecache.h>
#include <zenremotestore/builds/buildstorageutil.h>
#include <zenremotestore/builds/jupiterbuildstorage.h>
#include <zenremotestore/operationlogoutput.h>

#include <numeric>

namespace zen {

using namespace std::literals;

static const std::string_view OplogContainerPartName = "oplogcontainer"sv;

class BuildsRemoteStore : public RemoteProjectStore
{
public:
	BuildsRemoteStore(LoggerRef					   InLog,
					  const HttpClientSettings&	   ClientSettings,
					  HttpClientSettings*		   OptionalCacheClientSettings,
					  std::string_view			   HostUrl,
					  std::string_view			   CacheUrl,
					  const std::filesystem::path& TempFilePath,
					  WorkerThreadPool&			   CacheBackgroundWorkerPool,
					  std::string_view			   Namespace,
					  std::string_view			   Bucket,
					  const Oid&				   BuildId,
					  const IoBuffer&			   MetaData,
					  bool						   ForceDisableBlocks,
					  bool						   ForceDisableTempBlocks,
					  bool						   PopulateCache)
	: m_Log(InLog)
	, m_BuildStorageHttp(HostUrl, ClientSettings)
	, m_BuildStorage(CreateJupiterBuildStorage(Log(),
											   m_BuildStorageHttp,
											   m_BuildStorageStats,
											   Namespace,
											   Bucket,
											   /*AllowRedirect*/ false,
											   TempFilePath))
	, m_Namespace(Namespace)
	, m_Bucket(Bucket)
	, m_BuildId(BuildId)
	, m_MetaData(MetaData)
	, m_EnableBlocks(!ForceDisableBlocks)
	, m_UseTempBlocks(!ForceDisableTempBlocks)
	, m_PopulateCache(PopulateCache)
	{
		m_MetaData.MakeOwned();
		if (OptionalCacheClientSettings)
		{
			ZEN_ASSERT(!CacheUrl.empty());
			m_BuildCacheStorageHttp = std::make_unique<HttpClient>(CacheUrl, *OptionalCacheClientSettings);
			m_BuildCacheStorage		= CreateZenBuildStorageCache(*m_BuildCacheStorageHttp,
															 m_StorageCacheStats,
															 Namespace,
															 Bucket,
															 TempFilePath,
															 CacheBackgroundWorkerPool);
		}
	}

	virtual RemoteStoreInfo GetInfo() const override
	{
		return {.CreateBlocks	   = m_EnableBlocks,
				.UseTempBlockFiles = m_UseTempBlocks,
				.AllowChunking	   = true,
				.ContainerName	   = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId),
				.Description	   = fmt::format("[cloud] {}{}. SessionId: {}. {}/{}/{}"sv,
											 m_BuildStorageHttp.GetBaseUri(),
											 m_BuildCacheStorage ? fmt::format(" (Cache: {})", m_BuildCacheStorageHttp->GetBaseUri()) : ""sv,
											 m_BuildStorageHttp.GetSessionId(),
											 m_Namespace,
											 m_Bucket,
											 m_BuildId)};
	}

	virtual Stats GetStats() const override
	{
		return {
			.m_SentBytes	 = m_BuildStorageStats.TotalBytesWritten.load() + m_StorageCacheStats.TotalBytesWritten.load(),
			.m_ReceivedBytes = m_BuildStorageStats.TotalBytesRead.load() + m_StorageCacheStats.TotalBytesRead.load(),
			.m_RequestTimeNS = m_BuildStorageStats.TotalRequestTimeUs.load() * 1000 + m_StorageCacheStats.TotalRequestTimeUs.load() * 1000,
			.m_RequestCount	 = m_BuildStorageStats.TotalRequestCount.load() + m_StorageCacheStats.TotalRequestCount.load(),
			.m_PeakSentBytes = Max(m_BuildStorageStats.PeakSentBytes.load(), m_StorageCacheStats.PeakSentBytes.load()),
			.m_PeakReceivedBytes = Max(m_BuildStorageStats.PeakReceivedBytes.load(), m_StorageCacheStats.PeakReceivedBytes.load()),
			.m_PeakBytesPerSec	 = Max(m_BuildStorageStats.PeakBytesPerSec.load(), m_StorageCacheStats.PeakBytesPerSec.load()),
		};
	}

	virtual bool GetExtendedStats(ExtendedStats& OutStats) const override
	{
		bool								 Result = false;
		BuildStorageBase::ExtendedStatistics StorageStats;
		if (m_BuildStorage->GetExtendedStatistics(StorageStats))
		{
			for (auto It : StorageStats.ReceivedBytesPerSource)
			{
				OutStats.m_ReceivedBytesPerSource.insert_or_assign(It.first, It.second);
			}
			Result = true;
		}
		if (m_BuildCacheStorage)
		{
			OutStats.m_ReceivedBytesPerSource.insert_or_assign("Cache", m_StorageCacheStats.TotalBytesRead);
			Result = true;
		}
		return Result;
	}

	virtual CreateContainerResult CreateContainer() override
	{
		ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);

		CreateContainerResult Result;
		Stopwatch			  Timer;
		auto				  _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });

		CbObject Payload = LoadCompactBinaryObject(m_MetaData);
		try
		{
			CbObject PutBuildResult = m_BuildStorage->PutBuild(m_BuildId, Payload);
			ZEN_UNUSED(PutBuildResult);
			m_OplogBuildPartId = Oid::NewOid();
		}
		catch (const HttpClientError& Ex)
		{
			Result.ErrorCode = MakeErrorCode(Ex);
			Result.Reason	 = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}
		catch (const std::exception& Ex)
		{
			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
			Result.Reason	 = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}

		return Result;
	}

	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
	{
		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);

		SaveResult Result;
		Stopwatch  Timer;
		auto	   _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });

		try
		{
			CbObject ObjectPayload = LoadCompactBinaryObject(Payload);

			std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
				m_BuildStorage->PutBuildPart(m_BuildId, m_OplogBuildPartId, OplogContainerPartName, ObjectPayload);
			Result.RawHash = PutBuildPartResult.first;
			Result.Needs   = std::unordered_set<IoHash, IoHash::Hasher>(PutBuildPartResult.second.begin(), PutBuildPartResult.second.end());
		}
		catch (const HttpClientError& Ex)
		{
			Result.ErrorCode = MakeErrorCode(Ex);
			Result.Reason	 = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   m_OplogBuildPartId,
										   Ex.what());
		}
		catch (const std::exception& Ex)
		{
			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
			Result.Reason	 = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   m_OplogBuildPartId,
										   Ex.what());
		}

		return Result;
	}

	virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer&	Payload,
												const IoHash&			RawHash,
												ChunkBlockDescription&& Block) override
	{
		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);

		SaveAttachmentResult Result;
		Stopwatch			 Timer;
		auto				 _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });

		try
		{
			m_BuildStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);

			if (Block.BlockHash == RawHash)
			{
				try
				{
					CbObjectWriter BlockMetaData;
					BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string());
					CbObject MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save());
					if (!m_BuildStorage->PutBlockMetadata(m_BuildId, RawHash, MetaPayload))
					{
						ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'",
								 m_BuildStorageHttp.GetBaseUri(),
								 m_Namespace,
								 m_Bucket,
								 m_BuildId,
								 RawHash,
								 "not found");
					}
				}
				catch (const HttpClientError& Ex)
				{
					Result.ErrorCode = MakeErrorCode(Ex);
					Result.Reason	 = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'",
												   m_BuildStorageHttp.GetBaseUri(),
												   m_Namespace,
												   m_Bucket,
												   m_BuildId,
												   Ex.what());
				}
				catch (const std::exception& Ex)
				{
					Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
					Result.Reason	 = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'",
												   m_BuildStorageHttp.GetBaseUri(),
												   m_Namespace,
												   m_Bucket,
												   m_BuildId,
												   Ex.what());
				}
			}
		}
		catch (const HttpClientError& Ex)
		{
			Result.ErrorCode = MakeErrorCode(Ex);
			Result.Reason	 = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}
		catch (const std::exception& Ex)
		{
			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
			Result.Reason	 = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}

		return Result;
	}

	virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override
	{
		SaveAttachmentsResult Result;
		Stopwatch			  Timer;
		auto				  _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });
		for (const SharedBuffer& Chunk : Chunks)
		{
			CompressedBuffer	 Compressed	 = CompressedBuffer::FromCompressedNoValidate(Chunk.AsIoBuffer());
			SaveAttachmentResult ChunkResult = SaveAttachment(Compressed.GetCompressed(), Compressed.DecodeRawHash(), {});
			if (ChunkResult.ErrorCode)
			{
				return SaveAttachmentsResult{ChunkResult};
			}
		}
		return Result;
	}

	virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override
	{
		ZEN_UNUSED(RawHash);
		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);

		FinalizeResult Result;
		Stopwatch	   Timer;
		auto		   _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });

		try
		{
			std::vector<IoHash> Needs = m_BuildStorage->FinalizeBuildPart(m_BuildId, m_OplogBuildPartId, RawHash);
			Result.Needs			  = std::unordered_set<IoHash, IoHash::Hasher>(Needs.begin(), Needs.end());
		}
		catch (const HttpClientError& Ex)
		{
			Result.ErrorCode = Ex.GetInternalErrorCode() != 0							 ? Ex.GetInternalErrorCode()
							   : Ex.GetHttpResponseCode() != HttpResponseCode::ImATeapot ? (int)Ex.GetHttpResponseCode()
																						 : 0;
			Result.Reason	 = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   m_OplogBuildPartId,
										   Ex.what());
		}
		catch (const std::exception& Ex)
		{
			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
			Result.Reason	 = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   m_OplogBuildPartId,
										   Ex.what());
		}

		if (!Result.ErrorCode && Result.Needs.empty())
		{
			try
			{
				m_BuildStorage->FinalizeBuild(m_BuildId);
			}
			catch (const HttpClientError& Ex)
			{
				Result.ErrorCode = Ex.GetInternalErrorCode() != 0							 ? Ex.GetInternalErrorCode()
								   : Ex.GetHttpResponseCode() != HttpResponseCode::ImATeapot ? (int)Ex.GetHttpResponseCode()
																							 : 0;
				Result.Reason	 = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
											   m_BuildStorageHttp.GetBaseUri(),
											   m_Namespace,
											   m_Bucket,
											   m_BuildId,
											   Ex.what());
			}
			catch (const std::exception& Ex)
			{
				Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
				Result.Reason	 = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'",
											   m_BuildStorageHttp.GetBaseUri(),
											   m_Namespace,
											   m_Bucket,
											   m_BuildId,
											   Ex.what());
			}
		}

		return Result;
	}

	virtual LoadContainerResult LoadContainer() override
	{
		ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);

		LoadContainerResult Result;
		Stopwatch			Timer;
		auto				_ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });

		try
		{
			CbObject BuildObject = m_BuildStorage->GetBuild(m_BuildId);

			CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
			if (!PartsObject)
			{
				throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv,
													 m_BuildStorageHttp.GetBaseUri(),
													 m_Namespace,
													 m_Bucket,
													 m_BuildId));
			}
			m_OplogBuildPartId = PartsObject[OplogContainerPartName].AsObjectId();
			if (m_OplogBuildPartId == Oid::Zero)
			{
				throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv,
													 m_BuildStorageHttp.GetBaseUri(),
													 m_Namespace,
													 m_Bucket,
													 m_BuildId,
													 OplogContainerPartName));
			}

			Result.ContainerObject = m_BuildStorage->GetBuildPart(m_BuildId, m_OplogBuildPartId);
		}
		catch (const HttpClientError& Ex)
		{
			Result.ErrorCode = MakeErrorCode(Ex);
			Result.Reason	 = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}
		catch (const std::exception& Ex)
		{
			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
			Result.Reason	 = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}

		return Result;
	}

	virtual GetKnownBlocksResult GetKnownBlocks() override
	{
		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);

		GetKnownBlocksResult Result;
		Stopwatch			 Timer;
		auto				 _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });

		try
		{
			CbObject										  KnownBlocks = m_BuildStorage->FindBlocks(m_BuildId, 10000u);
			std::optional<std::vector<ChunkBlockDescription>> Blocks	  = ParseChunkBlockDescriptionList(KnownBlocks);
			Result.Blocks												  = std::move(Blocks.value());
		}
		catch (const HttpClientError& Ex)
		{
			Result.ErrorCode = MakeErrorCode(Ex);
			Result.Reason	 = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}
		catch (const std::exception& Ex)
		{
			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
			Result.Reason	 = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}

		return Result;
	}

	virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span<const IoHash> BlockHashes) override
	{
		std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(Log()));

		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);

		GetBlockDescriptionsResult Result;
		Stopwatch				   Timer;
		auto					   _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });

		try
		{
			Result.Blocks = zen::GetBlockDescriptions(*Output,
													  *m_BuildStorage,
													  m_BuildCacheStorage.get(),
													  m_BuildId,
													  m_OplogBuildPartId,
													  BlockHashes,
													  /*AttemptFallback*/ false,
													  /*IsQuiet*/ false,
													  /*IsVerbose)*/ false);
		}
		catch (const HttpClientError& Ex)
		{
			Result.ErrorCode = MakeErrorCode(Ex);
			Result.Reason	 = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}
		catch (const std::exception& Ex)
		{
			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
			Result.Reason	 = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}
		return Result;
	}

	virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span<const IoHash> RawHashes) override
	{
		AttachmentExistsInCacheResult Result;
		Stopwatch					  Timer;
		auto						  _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });
		try
		{
			const std::vector<BuildStorageCache::BlobExistsResult> CacheExistsResult =
				m_BuildCacheStorage->BlobsExists(m_BuildId, RawHashes);

			if (CacheExistsResult.size() == RawHashes.size())
			{
				Result.HasBody.reserve(CacheExistsResult.size());
				for (size_t BlobIndex = 0; BlobIndex < CacheExistsResult.size(); BlobIndex++)
				{
					Result.HasBody.push_back(CacheExistsResult[BlobIndex].HasBody);
				}
			}
		}
		catch (const HttpClientError& Ex)
		{
			Result.ErrorCode = MakeErrorCode(Ex);
			Result.Reason	 = fmt::format("Remote cache: Failed finding known blobs for {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}
		catch (const std::exception& Ex)
		{
			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
			Result.Reason	 = fmt::format("Remote cache: Failed finding known blobs for {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}
		return Result;
	}

	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) override
	{
		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);

		LoadAttachmentResult Result;
		Stopwatch			 Timer;
		auto				 _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });

		try
		{
			if (m_BuildCacheStorage)
			{
				IoBuffer CachedBlob = m_BuildCacheStorage->GetBuildBlob(m_BuildId, RawHash, Range.Offset, Range.Bytes);
				if (CachedBlob)
				{
					Result.Bytes = std::move(CachedBlob);
				}
			}
			if (!Result.Bytes)
			{
				Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash, Range.Offset, Range.Bytes);
				if (m_BuildCacheStorage && Result.Bytes && m_PopulateCache)
				{
					if (!Range)
					{
						m_BuildCacheStorage->PutBuildBlob(m_BuildId,
														  RawHash,
														  Result.Bytes.GetContentType(),
														  CompositeBuffer(SharedBuffer(Result.Bytes)));
					}
				}
			}
		}
		catch (const HttpClientError& Ex)
		{
			Result.ErrorCode = MakeErrorCode(Ex);
			Result.Reason	 = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}
		catch (const std::exception& Ex)
		{
			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
			Result.Reason	 = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'",
										   m_BuildStorageHttp.GetBaseUri(),
										   m_Namespace,
										   m_Bucket,
										   m_BuildId,
										   Ex.what());
		}

		return Result;
	}

	virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override
	{
		LoadAttachmentsResult Result;
		Stopwatch			  Timer;
		auto				  _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; });

		std::vector<IoHash> AttachmentsLeftToFind = RawHashes;

		if (m_BuildCacheStorage)
		{
			std::vector<BuildStorageCache::BlobExistsResult> ExistCheck = m_BuildCacheStorage->BlobsExists(m_BuildId, RawHashes);
			if (ExistCheck.size() == RawHashes.size())
			{
				AttachmentsLeftToFind.clear();
				for (size_t BlobIndex = 0; BlobIndex < RawHashes.size(); BlobIndex++)
				{
					const IoHash&							   Hash		  = RawHashes[BlobIndex];
					const BuildStorageCache::BlobExistsResult& BlobExists = ExistCheck[BlobIndex];
					if (BlobExists.HasBody)
					{
						IoBuffer CachedPayload = m_BuildCacheStorage->GetBuildBlob(m_BuildId, Hash);
						if (CachedPayload)
						{
							Result.Chunks.emplace_back(
								std::pair<IoHash, CompressedBuffer>{Hash,
																	CompressedBuffer::FromCompressedNoValidate(std::move(CachedPayload))});
						}
						else
						{
							AttachmentsLeftToFind.push_back(Hash);
						}
					}
					else
					{
						AttachmentsLeftToFind.push_back(Hash);
					}
				}
			}
		}

		for (const IoHash& Hash : AttachmentsLeftToFind)
		{
			LoadAttachmentResult ChunkResult = LoadAttachment(Hash, {});
			if (ChunkResult.ErrorCode)
			{
				return LoadAttachmentsResult{ChunkResult};
			}
			ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(ChunkResult.ElapsedSeconds * 1000)));
			if (m_BuildCacheStorage && ChunkResult.Bytes && m_PopulateCache)
			{
				m_BuildCacheStorage->PutBuildBlob(m_BuildId,
												  Hash,
												  ChunkResult.Bytes.GetContentType(),
												  CompositeBuffer(SharedBuffer(ChunkResult.Bytes)));
			}
			Result.Chunks.emplace_back(
				std::pair<IoHash, CompressedBuffer>{Hash, CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult.Bytes))});
		}
		return Result;
	}

	virtual void Flush() override
	{
		if (m_BuildCacheStorage)
		{
			m_BuildCacheStorage->Flush(100, [](intptr_t) { return false; });
		}
	}

private:
	static int MakeErrorCode(const HttpClientError& Ex)
	{
		return Ex.GetInternalErrorCode() != 0							 ? Ex.GetInternalErrorCode()
			   : Ex.GetHttpResponseCode() != HttpResponseCode::ImATeapot ? (int)Ex.GetHttpResponseCode()
																		 : 0;
	}

	inline LoggerRef Log() const { return m_Log; }

	LoggerRef m_Log;

	BuildStorageBase::Statistics	  m_BuildStorageStats;
	HttpClient						  m_BuildStorageHttp;
	std::unique_ptr<BuildStorageBase> m_BuildStorage;

	BuildStorageCache::Statistics	   m_StorageCacheStats;
	std::unique_ptr<HttpClient>		   m_BuildCacheStorageHttp;
	std::unique_ptr<BuildStorageCache> m_BuildCacheStorage;

	const std::string m_Namespace;
	const std::string m_Bucket;
	const Oid		  m_BuildId;
	IoBuffer		  m_MetaData;
	Oid				  m_OplogBuildPartId = Oid::Zero;
	const bool		  m_EnableBlocks	 = true;
	const bool		  m_UseTempBlocks	 = true;
	const bool		  m_AllowRedirect	 = false;
	const bool		  m_PopulateCache	 = true;
};

std::shared_ptr<RemoteProjectStore>
CreateJupiterBuildsRemoteStore(LoggerRef					   InLog,
							   const BuildsRemoteStoreOptions& Options,
							   const std::filesystem::path&	   TempFilePath,
							   bool							   Quiet,
							   bool							   Unattended,
							   bool							   Hidden,
							   WorkerThreadPool&			   CacheBackgroundWorkerPool,
							   double&						   OutHostLatencySec,
							   double&						   OutCacheLatencySec)
{
	std::string Host = Options.Host;
	if (!Host.empty() && Host.find("://"sv) == std::string::npos)
	{
		// Assume https URL
		Host = fmt::format("https://{}"sv, Host);
	}
	std::string OverrideUrl = Options.OverrideHost;
	if (!OverrideUrl.empty() && OverrideUrl.find("://"sv) == std::string::npos)
	{
		// Assume https URL
		OverrideUrl = fmt::format("https://{}"sv, OverrideUrl);
	}
	std::string ZenHost = Options.ZenHost;
	if (!ZenHost.empty() && ZenHost.find("://"sv) == std::string::npos)
	{
		// Assume https URL
		ZenHost = fmt::format("https://{}"sv, ZenHost);
	}

	// 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
	// 2) Access token as parameter in request
	// 3) Environment variable (different win vs linux/mac)
	// 4) Default openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider

	std::function<HttpClientAccessToken()> TokenProvider;
	if (!Options.OpenIdProvider.empty())
	{
		TokenProvider = httpclientauth::CreateFromOpenIdProvider(Options.AuthManager, Options.OpenIdProvider);
	}
	else if (!Options.AccessToken.empty())
	{
		TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken);
	}
	else if (!Options.OidcExePath.empty())
	{
		if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath,
																					Host.empty() ? OverrideUrl : Host,
																					Quiet,
																					Unattended,
																					Hidden);
			TokenProviderMaybe)
		{
			TokenProvider = TokenProviderMaybe.value();
		}
	}

	if (!TokenProvider)
	{
		TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
	}

	BuildStorageResolveResult ResolveRes;
	{
		HttpClientSettings ClientSettings{.LogCategory		   = "httpbuildsclient",
										  .AccessTokenProvider = TokenProvider,
										  .AssumeHttp2		   = Options.AssumeHttp2,
										  .AllowResume		   = true,
										  .RetryCount		   = 2};

		std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(InLog));

		ResolveRes =
			ResolveBuildStorage(*Output, ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery, /*Verbose*/ false);
	}

	HttpClientSettings ClientSettings{.LogCategory				   = "httpbuildsclient",
									  .ConnectTimeout			   = std::chrono::milliseconds(3000),
									  .Timeout					   = std::chrono::milliseconds(1800000),
									  .AccessTokenProvider		   = std::move(TokenProvider),
									  .AssumeHttp2				   = ResolveRes.HostAssumeHttp2,
									  .AllowResume				   = true,
									  .RetryCount				   = 4,
									  .MaximumInMemoryDownloadSize = Options.MaximumInMemoryDownloadSize};

	std::unique_ptr<HttpClientSettings> CacheClientSettings;

	if (!ResolveRes.CacheUrl.empty())
	{
		CacheClientSettings =
			std::make_unique<HttpClientSettings>(HttpClientSettings{.LogCategory				 = "httpcacheclient",
																	.ConnectTimeout				 = std::chrono::milliseconds{3000},
																	.Timeout					 = std::chrono::milliseconds{30000},
																	.AssumeHttp2				 = ResolveRes.CacheAssumeHttp2,
																	.AllowResume				 = true,
																	.RetryCount					 = 0,
																	.MaximumInMemoryDownloadSize = Options.MaximumInMemoryDownloadSize});
	}

	std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<BuildsRemoteStore>(InLog,
																						  ClientSettings,
																						  CacheClientSettings.get(),
																						  ResolveRes.HostUrl,
																						  ResolveRes.CacheUrl,
																						  TempFilePath,
																						  CacheBackgroundWorkerPool,
																						  Options.Namespace,
																						  Options.Bucket,
																						  Options.BuildId,
																						  Options.MetaData,
																						  Options.ForceDisableBlocks,
																						  Options.ForceDisableTempBlocks,
																						  Options.PopulateCache);

	OutHostLatencySec  = ResolveRes.HostLatencySec;
	OutCacheLatencySec = ResolveRes.CacheLatencySec;

	return RemoteStore;
}

}  // namespace zen