// Copyright Epic Games, Inc. All Rights Reserved. #include "buildsremoteprojectstore.h" #include #include #include #include #include #include namespace zen { using namespace std::literals; static const std::string_view OplogContainerPartName = "oplogcontainer"sv; class BuildsRemoteStore : public RemoteProjectStore { public: BuildsRemoteStore(std::unique_ptr&& BuildStorageStats, std::unique_ptr&& BuildStorageHttp, std::unique_ptr&& BuildStorage, std::string_view Url, std::string_view Namespace, std::string_view Bucket, const Oid& BuildId, const IoBuffer& MetaData, bool ForceDisableBlocks, bool ForceDisableTempBlocks) : m_BuildStorageStats(std::move(BuildStorageStats)) , m_BuildStorageHttp(std::move(BuildStorageHttp)) , m_BuildStorage(std::move(BuildStorage)) , m_Url(Url) , m_Namespace(Namespace) , m_Bucket(Bucket) , m_BuildId(BuildId) , m_MetaData(MetaData) , m_EnableBlocks(!ForceDisableBlocks) , m_UseTempBlocks(!ForceDisableTempBlocks) { m_MetaData.MakeOwned(); } virtual RemoteStoreInfo GetInfo() const override { return {.CreateBlocks = m_EnableBlocks, .UseTempBlockFiles = m_UseTempBlocks, .AllowChunking = true, .ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId), .Description = fmt::format("[cloud] {} as {}/{}/{}"sv, m_Url, m_Namespace, m_Bucket, m_BuildId)}; } virtual Stats GetStats() const override { return { .m_SentBytes = m_BuildStorageStats->TotalBytesWritten.load(), .m_ReceivedBytes = m_BuildStorageStats->TotalBytesRead.load(), .m_RequestTimeNS = m_BuildStorageStats->TotalRequestTimeUs.load() * 1000, .m_RequestCount = m_BuildStorageStats->TotalRequestCount.load(), .m_PeakSentBytes = m_BuildStorageStats->PeakSentBytes.load(), .m_PeakReceivedBytes = m_BuildStorageStats->PeakReceivedBytes.load(), .m_PeakBytesPerSec = m_BuildStorageStats->PeakBytesPerSec.load(), }; } virtual CreateContainerResult CreateContainer() override { ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero); CreateContainerResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); CbObject Payload = LoadCompactBinaryObject(m_MetaData); try { CbObject PutBuildResult = m_BuildStorage->PutBuild(m_BuildId, Payload); ZEN_UNUSED(PutBuildResult); m_OplogBuildPartId = Oid::NewOid(); } catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed creating oplog build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } return Result; } virtual SaveResult SaveContainer(const IoBuffer& Payload) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); SaveResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); try { CbObject ObjectPayload = LoadCompactBinaryObject(Payload); std::pair> PutBuildPartResult = m_BuildStorage->PutBuildPart(m_BuildId, m_OplogBuildPartId, OplogContainerPartName, ObjectPayload); Result.RawHash = PutBuildPartResult.first; Result.Needs = std::unordered_set(PutBuildPartResult.second.begin(), PutBuildPartResult.second.end()); } catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed saving oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, Ex.what()); } return Result; } virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&& Block) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); SaveAttachmentResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); try { m_BuildStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); if (Block.BlockHash == RawHash) { try { CbObjectWriter BlockMetaData; BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string()); CbObject MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()); if (!m_BuildStorage->PutBlockMetadata(m_BuildId, RawHash, MetaPayload)) { ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, RawHash, "not found"); } } catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed saving block attachment meta data to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } } } catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } return Result; } virtual SaveAttachmentsResult SaveAttachments(const std::vector& Chunks) override { SaveAttachmentsResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); for (const SharedBuffer& Chunk : Chunks) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(Chunk.AsIoBuffer()); SaveAttachmentResult ChunkResult = SaveAttachment(Compressed.GetCompressed(), Compressed.DecodeRawHash(), {}); if (ChunkResult.ErrorCode) { return SaveAttachmentsResult{ChunkResult}; } } return Result; } virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override { ZEN_UNUSED(RawHash); ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); FinalizeResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); try { std::vector Needs = m_BuildStorage->FinalizeBuildPart(m_BuildId, m_OplogBuildPartId, RawHash); Result.Needs = std::unordered_set(Needs.begin(), Needs.end()); } catch (const HttpClientError& Ex) { Result.ErrorCode = Ex.m_Error != 0 ? Ex.m_Error : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode : 0; Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed finalizing oplog container build part to {}/{}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, Ex.what()); } if (!Result.ErrorCode && Result.Needs.empty()) { try { m_BuildStorage->FinalizeBuild(m_BuildId); } catch (const HttpClientError& Ex) { Result.ErrorCode = Ex.m_Error != 0 ? Ex.m_Error : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode : 0; Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed finalizing oplog container build to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } } return Result; } virtual LoadContainerResult LoadContainer() override { ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero); LoadContainerResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); try { CbObject BuildObject = m_BuildStorage->GetBuild(m_BuildId); CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); if (!PartsObject) { throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload does not contain a 'parts' object"sv, m_Url, m_Namespace, m_Bucket, m_BuildId)); } m_OplogBuildPartId = PartsObject[OplogContainerPartName].AsObjectId(); if (m_OplogBuildPartId == Oid::Zero) { throw std::runtime_error(fmt::format("The build {}/{}/{}/{} payload 'parts' object does not contain a '{}' entry"sv, m_Url, m_Namespace, m_Bucket, m_BuildId, OplogContainerPartName)); } Result.ContainerObject = m_BuildStorage->GetBuildPart(m_BuildId, m_OplogBuildPartId); } catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed fetching oplog container build part to {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } return Result; } virtual GetKnownBlocksResult GetKnownBlocks() override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); GetKnownBlocksResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); try { CbObject KnownBlocks = m_BuildStorage->FindBlocks(m_BuildId, 10000u); std::optional> Blocks = ParseChunkBlockDescriptionList(KnownBlocks); Result.Blocks.reserve(Blocks.value().size()); for (ChunkBlockDescription& BlockDescription : Blocks.value()) { Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash, .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)}); } } catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } return Result; } virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); LoadAttachmentResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); try { Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash); } catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_Url, m_Namespace, m_Bucket, m_BuildId, Ex.what()); } return Result; } virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) override { LoadAttachmentsResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() * 1000000.0; }); for (const IoHash& Hash : RawHashes) { LoadAttachmentResult ChunkResult = LoadAttachment(Hash); if (ChunkResult.ErrorCode) { return LoadAttachmentsResult{ChunkResult}; } ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast(ChunkResult.ElapsedSeconds * 1000))); Result.Chunks.emplace_back( std::pair{Hash, CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult.Bytes))}); } return Result; } private: static int MakeErrorCode(const HttpClientError& Ex) { return Ex.m_Error != 0 ? Ex.m_Error : Ex.m_ResponseCode != HttpResponseCode::ImATeapot ? (int)Ex.m_ResponseCode : 0; } std::unique_ptr m_BuildStorageStats; std::unique_ptr m_BuildStorageHttp; std::unique_ptr m_BuildStorage; const std::string m_Url; const std::string m_Namespace; const std::string m_Bucket; const Oid m_BuildId; IoBuffer m_MetaData; Oid m_OplogBuildPartId = Oid::Zero; const bool m_EnableBlocks = true; const bool m_UseTempBlocks = true; const bool m_AllowRedirect = false; }; std::shared_ptr CreateJupiterBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath, bool Quiet) { std::string Url = Options.Url; if (Url.find("://"sv) == std::string::npos) { // Assume https URL Url = fmt::format("https://{}"sv, Url); } // 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider // 2) Access token as parameter in request // 3) Environment variable (different win vs linux/mac) // 4) Default openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider std::function TokenProvider; if (!Options.OpenIdProvider.empty()) { TokenProvider = httpclientauth::CreateFromOpenIdProvider(Options.AuthManager, Options.OpenIdProvider); } else if (!Options.AccessToken.empty()) { TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken); } else if (!Options.OidcExePath.empty()) { if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url, Quiet); TokenProviderMaybe) { TokenProvider = TokenProviderMaybe.value(); } } if (!TokenProvider) { TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager); } HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", .ConnectTimeout = std::chrono::milliseconds(2000), .Timeout = std::chrono::milliseconds(1800000), .AccessTokenProvider = std::move(TokenProvider), .AssumeHttp2 = Options.AssumeHttp2, .AllowResume = true, .RetryCount = 4}; std::unique_ptr BuildStorageStats(std::make_unique()); std::unique_ptr BuildStorageHttp = std::make_unique(Url, ClientSettings); std::unique_ptr BuildStorage = CreateJupiterBuildStorage(Log(), *BuildStorageHttp, *BuildStorageStats, Options.Namespace, Options.Bucket, false, TempFilePath); std::shared_ptr RemoteStore = std::make_shared(std::move(BuildStorageStats), std::move(BuildStorageHttp), std::move(BuildStorage), Url, Options.Namespace, Options.Bucket, Options.BuildId, Options.MetaData, Options.ForceDisableBlocks, Options.ForceDisableTempBlocks); return RemoteStore; } } // namespace zen